mirror of
https://github.com/euzu/tuliprox.git
synced 2026-10-28 01:32:27 +01:00
Merge branch 'develop' into feature/failover
This commit is contained in:
@@ -64,6 +64,8 @@ backend/app/Cargo.lock
|
||||
# only exist on a developer's own machine), so it is not project source.
|
||||
/.claude/
|
||||
/.mcp.json
|
||||
AGENTS.md
|
||||
GOAL.md
|
||||
|
||||
# Local refactoring measurements (clippy inventories, cargo metadata, timings).
|
||||
# These are engineering artifacts of the modularization work, not project docs.
|
||||
|
||||
Generated
+22
-21
@@ -1292,7 +1292,7 @@ checksum = "431a4c31778fde52b4400de34975f219eeca55cc829a9de157cd743a5b230ecb"
|
||||
|
||||
[[package]]
|
||||
name = "frontend"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"blake3",
|
||||
@@ -3795,7 +3795,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "shared"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"base64 0.23.1",
|
||||
@@ -4399,7 +4399,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"arc-swap",
|
||||
@@ -4453,6 +4453,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-stream",
|
||||
"tokio-tungstenite",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower-http",
|
||||
@@ -4479,7 +4480,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-auth"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"axum",
|
||||
@@ -4502,7 +4503,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-btree"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"crc32fast",
|
||||
@@ -4522,7 +4523,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-config-loader"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"blake3",
|
||||
@@ -4542,7 +4543,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-core"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"arc-swap",
|
||||
@@ -4593,7 +4594,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-curation"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"log",
|
||||
@@ -4610,7 +4611,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-dvr"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"blake3",
|
||||
@@ -4638,7 +4639,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-hls"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
@@ -4675,7 +4676,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-iptv"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
@@ -4700,7 +4701,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-library"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"bytes",
|
||||
@@ -4727,7 +4728,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-media-server"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -4746,7 +4747,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-messaging"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"chrono",
|
||||
@@ -4768,7 +4769,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-metadata"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"chrono",
|
||||
@@ -4791,7 +4792,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-mpegts"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"bytes",
|
||||
@@ -4805,7 +4806,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-parser"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
@@ -4825,7 +4826,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-processing"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"chrono",
|
||||
@@ -4864,7 +4865,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-repository"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"base64 0.23.1",
|
||||
@@ -4902,7 +4903,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-session"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"axum",
|
||||
@@ -4932,7 +4933,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tuliprox-testkit"
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"base64 0.23.1",
|
||||
|
||||
+1
-9
@@ -3,7 +3,7 @@ members = ["backend/*", "frontend", "shared", "tools/tuliprox-testkit"]
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "3.3.155"
|
||||
version = "3.3.156"
|
||||
edition = "2021"
|
||||
rust-version = "1.95.0"
|
||||
|
||||
@@ -24,14 +24,6 @@ return_self_not_must_use = "allow"
|
||||
missing_errors_doc = "allow"
|
||||
large_futures = "allow"
|
||||
|
||||
# Its suggested rewrite of `assert!(x.is_empty())` is
|
||||
# `assert_eq!(x, [] as [some::crate_root::Path; 0])`, which names a type that
|
||||
# is usually not in scope at the assertion — which is why `clippy --fix`
|
||||
# declines to apply it. The panic message improves; every one of the 39 call
|
||||
# sites gets materially harder to read. Reverse this if the suggestion is
|
||||
# ever made to use an in-scope type.
|
||||
assert_is_empty = "allow"
|
||||
|
||||
[workspace.dependencies]
|
||||
shared = { version = "3", path = "shared" }
|
||||
tuliprox-hls = { version = "3", path = "backend/hls" }
|
||||
|
||||
@@ -100,6 +100,7 @@ windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Stora
|
||||
vergen = { version = "10.0.3", features = ["build"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-tungstenite = "0.29.0"
|
||||
tuliprox-processing = { workspace = true, features = ["test-support"] }
|
||||
tuliprox-hls = { workspace = true, features = ["test-support"] }
|
||||
# The migration and playlist-stream tests build legacy v2 databases and
|
||||
|
||||
@@ -195,7 +195,7 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi
|
||||
.await;
|
||||
|
||||
assert_eq!(retry.admission.permission(), UserConnectionPermission::Allowed);
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -326,7 +326,7 @@ async fn resolve_admission_with_strategies_allows_other_channel_after_recent_evi
|
||||
let admission = result.admission;
|
||||
|
||||
assert_eq!(admission.permission(), UserConnectionPermission::Allowed);
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -458,7 +458,7 @@ async fn resolve_admission_with_strategies_does_not_suppress_different_session_o
|
||||
|
||||
assert_eq!(admission.permission(), UserConnectionPermission::Allowed);
|
||||
assert_eq!(grace_mode, None);
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -641,7 +641,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same
|
||||
|
||||
app_state.connection_manager.release_connection(&hls_addr).await;
|
||||
assert_eq!(app_state.active_users.user_connections(&user.username).await, 0);
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
|
||||
let mut close_rx = app_state.connection_manager.get_close_connection_channel();
|
||||
let result = resolve_admission_with_strategies(
|
||||
@@ -664,7 +664,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same
|
||||
|
||||
assert_eq!(admission.permission(), UserConnectionPermission::Allowed);
|
||||
assert_eq!(grace_mode, None);
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
assert_eq!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(100), close_rx.recv()).await.ok().and_then(Result::ok),
|
||||
Some(crate::api::model::CloseConnectionSignal::WithReason(
|
||||
|
||||
@@ -128,7 +128,7 @@ async fn create_stream_response_details_preserves_stored_headers_when_fallback_o
|
||||
socket_bound: false,
|
||||
})
|
||||
.await;
|
||||
assert!(!created.is_empty());
|
||||
assert_ne!(created.len(), 0);
|
||||
app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await;
|
||||
|
||||
let busy = app_state.active_provider.acquire_exact_connection_with_grace(
|
||||
@@ -233,7 +233,7 @@ async fn force_provider_stream_response_clears_stored_headers_after_fallback_ope
|
||||
socket_bound: false,
|
||||
})
|
||||
.await;
|
||||
assert!(!created.is_empty());
|
||||
assert_ne!(created.len(), 0);
|
||||
app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await;
|
||||
|
||||
let busy = app_state.active_provider.acquire_exact_connection_with_grace(
|
||||
|
||||
@@ -777,7 +777,7 @@ mod tests {
|
||||
let refreshed = refresh_forced_targets(Arc::clone(¤t), &sources_with(vec![target(1, "my-target")]));
|
||||
|
||||
assert!(!refreshed.enabled, "a reload must not enable a filter the operator never asked for");
|
||||
assert!(refreshed.target_names.is_empty());
|
||||
assert_eq!(refreshed.target_names.len(), 0);
|
||||
}
|
||||
|
||||
/// Target ids are positional, so a surviving name must come back with
|
||||
@@ -808,6 +808,6 @@ mod tests {
|
||||
let refreshed = refresh_forced_targets(forced(&["gone"]), &sources_with(vec![target(1, "other")]));
|
||||
|
||||
assert!(!refreshed.enabled, "an empty filter must not be left enabled: it would process nothing");
|
||||
assert!(refreshed.target_names.is_empty());
|
||||
assert_eq!(refreshed.target_names.len(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ use serde::Deserialize;
|
||||
use std::{str::FromStr, sync::Arc};
|
||||
use tuliprox_hls::api::{
|
||||
finite_hls_immutable_media_response, resolve_hls_standalone_custom_segment, HlsAccessLeaseId,
|
||||
HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError,
|
||||
HlsMediaActivityCommitOutcome, HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError,
|
||||
};
|
||||
use url::form_urlencoded;
|
||||
|
||||
@@ -286,7 +286,9 @@ async fn cvs_standalone_hls_segment_response(
|
||||
asset_fingerprint,
|
||||
index,
|
||||
now_ms,
|
||||
) {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(access) => access,
|
||||
Err(
|
||||
HlsStandaloneCustomSegmentError::InvalidIndex
|
||||
@@ -299,13 +301,28 @@ async fn cvs_standalone_hls_segment_response(
|
||||
{
|
||||
return *response;
|
||||
}
|
||||
finite_hls_immutable_media_response(
|
||||
access.bytes,
|
||||
let response = finite_hls_immutable_media_response(
|
||||
access.bytes.clone(),
|
||||
headers.get(header::RANGE),
|
||||
HLS_CVS_CONTENT_TYPE,
|
||||
HLS_CVS_IMMUTABLE_CACHE_CONTROL,
|
||||
method == Method::HEAD,
|
||||
)
|
||||
);
|
||||
if response.status().is_success() {
|
||||
match app_state
|
||||
.hls
|
||||
.proxy
|
||||
.record_standalone_custom_segment_activity(&access_lease, &access, current_time_millis())
|
||||
.await
|
||||
{
|
||||
HlsMediaActivityCommitOutcome::Committed => {}
|
||||
HlsMediaActivityCommitOutcome::StaleLeaseIdentity => return StatusCode::NOT_FOUND.into_response(),
|
||||
HlsMediaActivityCommitOutcome::DeferredLockContention => {
|
||||
return super::hls_terminal_response::hls_temporary_resource_unavailable_response(1_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
async fn validate_hls_standalone_custom_access(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::{
|
||||
super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions},
|
||||
clear_hls_provisioning_handoff_consumer, duration_to_millis_saturating,
|
||||
effective_hls_url_failover_provider_for_fetch_url, hls_access_lease_timing_for_session, hls_access_lease_ttl_ms,
|
||||
hls_availability_reevaluation_registration_failure_response, hls_cache_configured,
|
||||
@@ -29,8 +30,7 @@ use tuliprox_hls::api::{
|
||||
safe_proxy_session_id, safe_session_key, HlsAccessContext, HlsAccessLeaseId, HlsAccessLeasePendingDeadline,
|
||||
HlsAccessLeaseState, HlsAccessLeaseTouch, HlsAccountBindingProtection, HlsManifestAcceptanceDirective,
|
||||
HlsManifestAcceptanceEvaluationOutcome, HlsManifestCommitRequirement, HlsOriginIoContext, HlsOriginWorkClass,
|
||||
HlsPostRefreshRuntime, HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry,
|
||||
OriginRefreshRequest, ProxySessionId, RetryPolicy,
|
||||
HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry, ProxySessionId,
|
||||
};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -251,13 +251,10 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
prepared_origin.runtime_provider_config.clone(),
|
||||
)?;
|
||||
{
|
||||
let mut session_guard = session.write().await;
|
||||
let session_guard = session.read().await;
|
||||
if session_guard.is_gc_marked_for_removal() {
|
||||
return Some(hls_canonical_retry_after_response());
|
||||
}
|
||||
if prepared_origin.origin_account_binding_to_store.is_some() {
|
||||
session_guard.replace_origin_account_binding(prepared_origin.origin_account_binding_to_store);
|
||||
}
|
||||
}
|
||||
mark_hls_authorized_manifest_access(app_state, &session, now_ms).await;
|
||||
let selected_account = session.read().await.origin_account_binding.as_ref().map_or_else(
|
||||
@@ -298,7 +295,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
(HlsManifestAcceptanceDirective::none(), Some(owner_key))
|
||||
}
|
||||
HlsManifestAcceptanceEvaluationOutcome::SessionSuperseded => {
|
||||
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
drop(prepared_origin.preacquired_origin_account_handle);
|
||||
return Some(hls_canonical_retry_after_response());
|
||||
}
|
||||
};
|
||||
@@ -317,7 +314,6 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
};
|
||||
|
||||
let origin_policy = hls_effective_origin_acquire_policy(&session).await;
|
||||
let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone();
|
||||
let mut preacquired_provider_handle = prepared_origin.preacquired_origin_account_handle;
|
||||
let mut origin_io = HlsOriginIoContext {
|
||||
ctx: hls_ctx.clone(),
|
||||
@@ -328,49 +324,37 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
reservation_ttl_secs,
|
||||
preacquired_provider_handle: None,
|
||||
started_generation: None,
|
||||
prepared_account: None,
|
||||
playback_kind: session.read().await.origin_playback_kind(),
|
||||
account_runtime: session.read().await.origin_account_runtime.clone(),
|
||||
};
|
||||
if availability_reevaluation_owner_key.is_none() {
|
||||
if let Some(provider_handle) = preacquired_provider_handle.take() {
|
||||
origin_io = origin_io.with_preacquired_provider_handle(provider_handle);
|
||||
origin_io = origin_io.with_managed_provider_handle(provider_handle);
|
||||
}
|
||||
}
|
||||
|
||||
let refresh_request = OriginRefreshRequest {
|
||||
app_config: Arc::clone(&app_state.app_config),
|
||||
session: Arc::clone(&session),
|
||||
origin_entry,
|
||||
headers,
|
||||
origin_provider_session_headers,
|
||||
client: app_state.http_clients.default.load().as_ref().clone(),
|
||||
no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(),
|
||||
use_manual_redirects: app_state.should_use_manual_redirects(),
|
||||
segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()),
|
||||
hls_proxy: Arc::clone(&app_state.hls.proxy),
|
||||
segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()),
|
||||
segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()),
|
||||
map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()),
|
||||
origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(),
|
||||
manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(),
|
||||
strip: app_state.hls.proxy.strip().clone(),
|
||||
retry_policy: RetryPolicy::default(),
|
||||
reverse_proxy_rewrite_secret: rewrite_secret.to_vec(),
|
||||
transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(),
|
||||
manifest_commit_requirement,
|
||||
fresh_manifest_requirement_generation: None,
|
||||
acceptance_directive,
|
||||
access_lease_id: Some(access_lease_id.clone()),
|
||||
disabled_headers: app_state.get_disabled_headers(),
|
||||
now_ms,
|
||||
origin_io: Some(origin_io),
|
||||
post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: hls_ctx.downgrade() }),
|
||||
};
|
||||
let refresh_request = hls_origin_refresh_request(
|
||||
app_state,
|
||||
&session,
|
||||
tuliprox_hls::api::HlsPreparedOriginAccount { entry: origin_entry, snapshot: prepared_origin.snapshot },
|
||||
HlsOriginRefreshOptions {
|
||||
headers,
|
||||
origin_io,
|
||||
manifest_commit_requirement,
|
||||
acceptance_directive,
|
||||
access_lease_id: Some(access_lease_id.clone()),
|
||||
rewrite_secret: rewrite_secret.to_vec(),
|
||||
now_ms,
|
||||
},
|
||||
);
|
||||
let refresh_ordering = if session_outcome == HlsSessionStoreOutcome::Reused {
|
||||
refresh_ordering
|
||||
} else {
|
||||
HlsManifestRefreshOrdering::Background
|
||||
};
|
||||
if let Some(owner_key) = availability_reevaluation_owner_key {
|
||||
app_state.connection_manager.release_provider_handle(preacquired_provider_handle);
|
||||
drop(preacquired_provider_handle);
|
||||
touch_initial_manifest_access_lease_window(
|
||||
app_state,
|
||||
access_lease_id,
|
||||
|
||||
@@ -217,7 +217,7 @@ pub(in crate::api::endpoints::hls_api) async fn evaluate_hls_canonical_owner_han
|
||||
return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable);
|
||||
}
|
||||
match &lease.playback_mode {
|
||||
HlsLeasePlaybackMode::TerminalTail(_) => {
|
||||
HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalCustomResponse(_) => {
|
||||
let Some(response) =
|
||||
hls_terminal_playback_response(&lease, context.proxy_session_id, context.access_lease_id)
|
||||
else {
|
||||
|
||||
@@ -516,6 +516,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_plans(
|
||||
pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry(
|
||||
plan: SharedHlsProvisioningSegmentPlan,
|
||||
content_length: u64,
|
||||
media_start_ms: u64,
|
||||
duration_ms: u64,
|
||||
now_ms: u64,
|
||||
) -> SegmentEntry {
|
||||
@@ -531,6 +532,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry(
|
||||
host_local_index: u32::try_from(plan.proxy_seq).unwrap_or(u32::MAX),
|
||||
},
|
||||
proxy_seq: plan.proxy_seq,
|
||||
media_start_ms,
|
||||
duration_ms,
|
||||
proxy_file_ext: "ts".to_string(),
|
||||
content_type: "video/mp2t".to_string(),
|
||||
@@ -636,10 +638,13 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_shared_hls_provisioning_h
|
||||
session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq);
|
||||
}
|
||||
session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq);
|
||||
let media_start_ms = session_guard.next_media_start_ms;
|
||||
session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms);
|
||||
session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1));
|
||||
session_guard
|
||||
.segments
|
||||
.insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms));
|
||||
session_guard.segments.insert(
|
||||
plan.proxy_seq,
|
||||
shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms),
|
||||
);
|
||||
inserted = true;
|
||||
}
|
||||
if inserted {
|
||||
@@ -682,10 +687,12 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline
|
||||
session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq);
|
||||
}
|
||||
session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq);
|
||||
let media_start_ms = session_guard.next_media_start_ms;
|
||||
session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms);
|
||||
session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1));
|
||||
session_guard.segments.insert(
|
||||
plan.proxy_seq,
|
||||
shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms),
|
||||
shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms),
|
||||
);
|
||||
}
|
||||
session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS);
|
||||
|
||||
@@ -110,10 +110,12 @@ pub(super) struct HlsProxyResourcePathParams {
|
||||
|
||||
mod catchup;
|
||||
mod manifest;
|
||||
mod origin_account_runtime;
|
||||
mod origin_refresh_request;
|
||||
mod owner_token;
|
||||
mod segment;
|
||||
mod session;
|
||||
|
||||
use self::origin_account_runtime::prepare_hls_origin_runtime;
|
||||
pub(in crate::api) use catchup::*;
|
||||
pub(in crate::api) use manifest::*;
|
||||
pub(in crate::api) use owner_token::*;
|
||||
|
||||
@@ -0,0 +1,719 @@
|
||||
#![allow(clippy::wildcard_imports)]
|
||||
use super::{
|
||||
origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions},
|
||||
session::*,
|
||||
*,
|
||||
};
|
||||
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub(super) async fn prepare_hls_origin_runtime(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
input: &ConfigInput,
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
proxy_session_id: &ProxySessionId,
|
||||
fingerprint: &Fingerprint,
|
||||
connection_kind: crate::api::model::ConnectionKind,
|
||||
priority: i8,
|
||||
work_kind: HlsOriginWorkKind,
|
||||
work_class: HlsOriginWorkClass,
|
||||
now_ms: u64,
|
||||
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
|
||||
{
|
||||
let current = session.read().await;
|
||||
if current.origin_source.input_id != input.id
|
||||
|| (!current.origin_source.input_name.is_empty() && current.origin_source.input_name != input.name)
|
||||
|| current.origin_account_binding.as_ref().is_some_and(|binding| binding.input_name != input.name)
|
||||
{
|
||||
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
|
||||
reason: hls_no_account_reason_for_binding(current.origin_account_binding.as_ref()),
|
||||
});
|
||||
}
|
||||
}
|
||||
advance_hls_origin_account_lifecycle(app_state, session, now_ms).await;
|
||||
// Healthy preparation does not queue behind account replacement. Preserve the manifest
|
||||
// singleflight boundary when many clients arrive while a qualified window awaits media.
|
||||
if let Some(current_input) =
|
||||
app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name)
|
||||
{
|
||||
let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await);
|
||||
let binding = observed.binding.clone();
|
||||
if let Some(binding) = binding.filter(|binding| {
|
||||
binding.is_active()
|
||||
&& hls_origin_account_status(&app_state.hls_ctx(), binding) == HlsOriginAccountStatus::Known
|
||||
}) {
|
||||
let prepared = prepared_hls_origin_runtime_for_known_binding(
|
||||
app_state,
|
||||
¤t_input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
&binding,
|
||||
)
|
||||
.await;
|
||||
let mut current = session.write().await;
|
||||
if !current.is_gc_marked_for_removal() && observed.matches_session(¤t) {
|
||||
if current.origin_account_runtime.is_none() {
|
||||
current.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime {
|
||||
app_state: Arc::downgrade(app_state),
|
||||
input_id: input.id,
|
||||
input_name: Arc::clone(&input.name),
|
||||
raw_request_url: raw_request_url.to_string(),
|
||||
entry_url: session_entry_url.to_string(),
|
||||
fingerprint: fingerprint.clone(),
|
||||
}));
|
||||
}
|
||||
return Ok(PreparedHlsOriginRuntime::published(
|
||||
prepared,
|
||||
HlsOriginAccountSnapshot::from_session(¤t),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock);
|
||||
let _prepare_guard = prepare_lock.lock().await;
|
||||
let Some(input) = app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name)
|
||||
else {
|
||||
let mut session = session.write().await;
|
||||
if session.origin_account_rebind.is_allowed_now(now_ms) {
|
||||
if let Some(binding) = session.origin_account_binding.clone() {
|
||||
session.origin_account_rebind.mark_attempt_started(binding.account_name, now_ms);
|
||||
session.origin_account_rebind.mark_failed(now_ms);
|
||||
}
|
||||
}
|
||||
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
|
||||
reason: hls_no_account_reason_for_binding(session.origin_account_binding.as_ref()),
|
||||
});
|
||||
};
|
||||
session.write().await.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime {
|
||||
app_state: Arc::downgrade(app_state),
|
||||
input_id: input.id,
|
||||
input_name: Arc::clone(&input.name),
|
||||
raw_request_url: raw_request_url.to_string(),
|
||||
entry_url: session_entry_url.to_string(),
|
||||
fingerprint: fingerprint.clone(),
|
||||
}));
|
||||
let input = input.as_ref();
|
||||
advance_hls_origin_account_lifecycle(app_state, session, now_ms).await;
|
||||
let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await);
|
||||
let previous = observed.binding.clone();
|
||||
let mut prepared = prepare_hls_origin_runtime_locked(
|
||||
app_state,
|
||||
session,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
work_kind,
|
||||
work_class,
|
||||
now_ms,
|
||||
)
|
||||
.await?;
|
||||
let snapshot = publish_hls_origin_account_with_status(app_state, session, |current| {
|
||||
let snapshot = if let Some(snapshot) = prepared.prepared_snapshot.take() {
|
||||
if current.is_gc_marked_for_removal() || !snapshot.matches_session(current) {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
snapshot
|
||||
} else {
|
||||
if current.is_gc_marked_for_removal() || !observed.matches_session(current) {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
if let Some(mut binding) = prepared.binding_to_publish.take() {
|
||||
binding.generation = previous.as_ref().map_or(0, |previous| previous.generation.saturating_add(1));
|
||||
binding.provider_binding_tag = prepared
|
||||
.preacquired_origin_account_handle
|
||||
.as_ref()
|
||||
.and_then(tuliprox_session::ManagedProviderHandle::handle)
|
||||
.and_then(|handle| handle.binding_tag);
|
||||
if previous.as_ref().is_some_and(|previous| previous.account_name != binding.account_name) {
|
||||
current.handoff_origin_account(binding);
|
||||
} else {
|
||||
current.replace_origin_account_binding(Some(binding));
|
||||
}
|
||||
}
|
||||
HlsOriginAccountSnapshot::from_session(current)
|
||||
};
|
||||
Ok(snapshot)
|
||||
})
|
||||
.await?;
|
||||
Ok(PreparedHlsOriginRuntime::published(prepared, snapshot))
|
||||
}
|
||||
|
||||
// Lock order: viewer rows, then HLS state. Publication and projection are synchronous once
|
||||
// both locks are acquired: cancellation cannot publish an account without publishing its UI rows.
|
||||
pub(super) async fn publish_hls_origin_account_with_status(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
publish: impl FnOnce(
|
||||
&mut tuliprox_hls::api::HlsSession,
|
||||
) -> Result<HlsOriginAccountSnapshot, HlsOriginRuntimeAcquireError>,
|
||||
) -> Result<HlsOriginAccountSnapshot, HlsOriginRuntimeAcquireError> {
|
||||
let shared_stream_id = hls_cache_shared_stream_id(&session.read().await.proxy_session_id);
|
||||
let mut projection = app_state.active_users.lock_shared_stream_provider_projection(shared_stream_id).await;
|
||||
let mut current = session.write().await;
|
||||
let snapshot = publish(&mut current)?;
|
||||
if !snapshot.matches_session(¤t) {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
if let Some(binding) = snapshot.binding.as_ref().filter(|binding| binding.is_active()) {
|
||||
projection.update_provider(&binding.account_name);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn advance_hls_origin_account_lifecycle(app_state: &Arc<AppState>, session: &HlsSessionHandle, now_ms: u64) {
|
||||
promote_elapsed_hls_account_overlaps(app_state, now_ms).await;
|
||||
detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await;
|
||||
reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await;
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
async fn prepare_hls_origin_runtime_locked(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
input: &ConfigInput,
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
proxy_session_id: &ProxySessionId,
|
||||
fingerprint: &Fingerprint,
|
||||
connection_kind: crate::api::model::ConnectionKind,
|
||||
priority: i8,
|
||||
work_kind: HlsOriginWorkKind,
|
||||
work_class: HlsOriginWorkClass,
|
||||
now_ms: u64,
|
||||
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
|
||||
let playback_kind = hls_session_origin_playback_kind(session).await;
|
||||
let existing_binding = session.read().await.origin_account_binding.clone();
|
||||
let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached);
|
||||
if reacquire_detached_binding && !session.read().await.origin_account_rebind.is_allowed_now(now_ms) {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref());
|
||||
if reacquire_detached_binding {
|
||||
log_hls_origin_binding_reacquire_started(session, work_kind).await;
|
||||
}
|
||||
if let Some(binding) = existing_binding {
|
||||
if binding.is_active()
|
||||
|| !matches!(hls_origin_account_status(&app_state.hls_ctx(), &binding), HlsOriginAccountStatus::Known)
|
||||
{
|
||||
match hls_origin_account_status(&app_state.hls_ctx(), &binding) {
|
||||
stale_status @ (HlsOriginAccountStatus::Missing
|
||||
| HlsOriginAccountStatus::Expired
|
||||
| HlsOriginAccountStatus::Blocked) => {
|
||||
return rebind_hls_origin_account(
|
||||
app_state,
|
||||
session,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
&binding,
|
||||
stale_status,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
HlsOriginAccountStatus::Known => {
|
||||
return Ok(prepared_hls_origin_runtime_for_known_binding(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
&binding,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match prepare_hls_origin_runtime_with_new_account(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
false,
|
||||
work_kind,
|
||||
work_class,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => {
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.binding_to_publish.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)),
|
||||
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
|
||||
}
|
||||
|
||||
if work_class.allows_speculative_overlap() {
|
||||
if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.binding_to_publish.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
|
||||
if let Ok(prepared) = prepare_hls_speculative_origin_runtime(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.binding_to_publish.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
} else {
|
||||
debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value());
|
||||
}
|
||||
|
||||
if work_class.allows_grace() {
|
||||
match prepare_hls_origin_runtime_with_new_account(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
true,
|
||||
work_kind,
|
||||
work_class,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => {
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.binding_to_publish.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(status))
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
"HLS origin account grace skipped: work_class={} reason=background-origin-work",
|
||||
work_class.as_log_value()
|
||||
);
|
||||
}
|
||||
|
||||
if reacquire_detached_binding {
|
||||
log_hls_origin_binding_reacquire_failed(session, "no-account-available").await;
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason })
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[allow(clippy::too_many_lines)]
|
||||
pub(super) async fn rebind_hls_origin_account(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
input: &ConfigInput,
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
stale_binding: &HlsOriginAccountBinding,
|
||||
stale_status: HlsOriginAccountStatus,
|
||||
fingerprint: &Fingerprint,
|
||||
connection_kind: crate::api::model::ConnectionKind,
|
||||
priority: i8,
|
||||
playback_kind: PlaybackKind,
|
||||
now_ms: u64,
|
||||
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
|
||||
{
|
||||
let mut session_guard = session.write().await;
|
||||
if !session_guard.origin_account_rebind.is_allowed_now(now_ms) {
|
||||
debug!(
|
||||
"HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000",
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id),
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref())
|
||||
);
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms);
|
||||
}
|
||||
|
||||
let safe_proxy_session = {
|
||||
let session_guard = session.read().await;
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id)
|
||||
};
|
||||
debug!(
|
||||
"HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}",
|
||||
safe_proxy_session,
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref())
|
||||
);
|
||||
app_state.active_provider.clear_identified_provider_reservation(
|
||||
&stale_binding.session_owner,
|
||||
&stale_binding.account_name,
|
||||
stale_binding.provider_binding_tag,
|
||||
);
|
||||
let expected_work_generation = {
|
||||
let mut session_guard = session.write().await;
|
||||
if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| {
|
||||
binding.account_name == stale_binding.account_name
|
||||
&& binding.session_owner == stale_binding.session_owner
|
||||
&& binding.provider_binding_tag == stale_binding.provider_binding_tag
|
||||
}) {
|
||||
binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms);
|
||||
debug!(
|
||||
"HLS origin binding detached: proxy_session={} account={} reason={}",
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id),
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
|
||||
HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason()
|
||||
);
|
||||
}
|
||||
session_guard.invalidate_queued_origin_work();
|
||||
session_guard.activity.origin_work_generation
|
||||
};
|
||||
|
||||
let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session(
|
||||
&input.name,
|
||||
&fingerprint.addr,
|
||||
false,
|
||||
priority,
|
||||
connection_kind,
|
||||
Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)),
|
||||
) else {
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
|
||||
reason: hls_no_account_reason_for_binding(Some(stale_binding)),
|
||||
});
|
||||
};
|
||||
|
||||
let managed_handle =
|
||||
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
|
||||
let Some(provider_config) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_config())
|
||||
else {
|
||||
drop(managed_handle);
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
let Some(fetch_url) = resolve_hls_origin_fetch_url(
|
||||
&app_state.app_config,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
Some(&provider_config),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
drop(managed_handle);
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
let Some(new_account_name) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_name())
|
||||
else {
|
||||
drop(managed_handle);
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
let mut new_binding = HlsOriginAccountBinding::rebound(
|
||||
Arc::clone(&input.name),
|
||||
new_account_name,
|
||||
stale_binding.session_owner.clone(),
|
||||
stale_binding.generation.saturating_add(1),
|
||||
now_ms,
|
||||
);
|
||||
new_binding.provider_binding_tag = managed_handle.handle().and_then(|handle| handle.binding_tag);
|
||||
let prepared_snapshot = publish_hls_origin_account_with_status(app_state, session, |session_guard| {
|
||||
if !publish_hls_origin_account_rebind(
|
||||
session_guard,
|
||||
stale_binding,
|
||||
expected_work_generation,
|
||||
new_binding.clone(),
|
||||
) {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
Ok(HlsOriginAccountSnapshot::from_session(session_guard))
|
||||
})
|
||||
.await?;
|
||||
debug!(
|
||||
"HLS origin account rebound: old_account={} new_account={}",
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
|
||||
sanitize_sensitive_info(new_binding.account_name.as_ref())
|
||||
);
|
||||
|
||||
Ok(HlsOriginRuntimeAllocation {
|
||||
url_failover_provider: hls_url_failover_provider_for_origin_context(
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
&fetch_url,
|
||||
),
|
||||
runtime_provider_config: Some(provider_config),
|
||||
fetch_url,
|
||||
binding_to_publish: None,
|
||||
prepared_snapshot: Some(prepared_snapshot),
|
||||
preacquired_origin_account_handle: Some(managed_handle),
|
||||
})
|
||||
}
|
||||
|
||||
// Resolution may suspend after detaching. Publish only into the same account and work generation.
|
||||
fn publish_hls_origin_account_rebind(
|
||||
session: &mut tuliprox_hls::api::HlsSession,
|
||||
expected: &HlsOriginAccountBinding,
|
||||
expected_work_generation: u64,
|
||||
replacement: HlsOriginAccountBinding,
|
||||
) -> bool {
|
||||
if session.is_gc_marked_for_removal()
|
||||
|| session.activity.origin_work_generation != expected_work_generation
|
||||
|| !session.origin_account_binding.as_ref().is_some_and(|current| {
|
||||
current.input_name == expected.input_name
|
||||
&& current.account_name == expected.account_name
|
||||
&& current.session_owner == expected.session_owner
|
||||
&& current.generation == expected.generation
|
||||
&& current.provider_binding_tag == expected.provider_binding_tag
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
session.handoff_origin_account(replacement);
|
||||
session.origin_account_rebind.mark_success();
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) async fn mark_hls_origin_rebind_failed(
|
||||
session: &HlsSessionHandle,
|
||||
stale_binding: &HlsOriginAccountBinding,
|
||||
now_ms: u64,
|
||||
reason: &str,
|
||||
) {
|
||||
let mut session_guard = session.write().await;
|
||||
session_guard.origin_account_rebind.mark_failed(now_ms);
|
||||
debug!(
|
||||
"HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000",
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id),
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref())
|
||||
);
|
||||
}
|
||||
|
||||
use std::{future::Future, pin::Pin, sync::Weak};
|
||||
use tuliprox_hls::api::{
|
||||
trigger_origin_refresh_sync, HlsBoundAccountAcquireErrorKind, HlsManifestAcceptanceDirective,
|
||||
HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount,
|
||||
LiveHlsOriginEntry,
|
||||
};
|
||||
|
||||
struct AppHlsOriginAccountRuntime {
|
||||
app_state: Weak<AppState>,
|
||||
input_id: u16,
|
||||
input_name: Arc<str>,
|
||||
raw_request_url: String,
|
||||
entry_url: String,
|
||||
fingerprint: Fingerprint,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AppHlsOriginAccountRuntime {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("AppHlsOriginAccountRuntime").field("input_id", &self.input_id).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsOriginAccountRuntime for AppHlsOriginAccountRuntime {
|
||||
#[allow(clippy::too_many_lines)]
|
||||
fn prepare<'a>(
|
||||
&'a self,
|
||||
session: &'a HlsSessionHandle,
|
||||
work: HlsOriginAccountWork,
|
||||
class: HlsOriginWorkClass,
|
||||
) -> Pin<Box<dyn Future<Output = Result<HlsPreparedOriginAccount, HlsBoundAccountAcquireErrorKind>> + Send + 'a>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
let app_state = self.app_state.upgrade().ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?;
|
||||
let input = app_state
|
||||
.app_config
|
||||
.get_input_by_id(self.input_id)
|
||||
.filter(|input| input.name == self.input_name)
|
||||
.ok_or(HlsBoundAccountAcquireErrorKind::Missing)?;
|
||||
let (proxy_id, policy, unavailable) = {
|
||||
let session = session.read().await;
|
||||
(
|
||||
session.proxy_session_id.clone(),
|
||||
session.effective_origin_acquire_policy_or_default(),
|
||||
session.is_gc_marked_for_removal(),
|
||||
)
|
||||
};
|
||||
if unavailable
|
||||
|| !app_state
|
||||
.hls
|
||||
.proxy
|
||||
.segment_worker_pool()
|
||||
.access_leases()
|
||||
.write()
|
||||
.await
|
||||
.has_usable_access_lease_for_session(&proxy_id, current_time_millis())
|
||||
{
|
||||
return Err(HlsBoundAccountAcquireErrorKind::Unavailable);
|
||||
}
|
||||
let prepared = prepare_hls_origin_runtime(
|
||||
&app_state,
|
||||
session,
|
||||
&input,
|
||||
&self.raw_request_url,
|
||||
&self.entry_url,
|
||||
&proxy_id,
|
||||
&self.fingerprint,
|
||||
policy.connection_kind,
|
||||
policy.priority,
|
||||
if work == HlsOriginAccountWork::Manifest {
|
||||
HlsOriginWorkKind::Manifest
|
||||
} else {
|
||||
HlsOriginWorkKind::Resource
|
||||
},
|
||||
class,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| HlsBoundAccountAcquireErrorKind::Unavailable)?;
|
||||
// Drop releases unused allocations synchronously, including cancellation after prepare.
|
||||
let managed = prepared.preacquired_origin_account_handle;
|
||||
let entry = LiveHlsOriginEntry::parse_with_provider_configs(
|
||||
&prepared.fetch_url,
|
||||
prepared.url_failover_provider,
|
||||
prepared.runtime_provider_config,
|
||||
)
|
||||
.ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?;
|
||||
let snapshot = prepared.snapshot;
|
||||
let (pending, origin_headers) = {
|
||||
let current = session.read().await;
|
||||
if current.is_gc_marked_for_removal() || !snapshot.matches_session(¤t) {
|
||||
return Err(HlsBoundAccountAcquireErrorKind::StoreRace);
|
||||
}
|
||||
(current.origin_account_manifest_pending.is_some(), current.origin_request_headers.clone())
|
||||
};
|
||||
drop(managed);
|
||||
if work == HlsOriginAccountWork::Media && pending {
|
||||
let origin_io = HlsOriginIoContext {
|
||||
ctx: app_state.hls_ctx(),
|
||||
client_addr: self.fingerprint.addr,
|
||||
allow_grace: class.allows_grace(),
|
||||
priority: policy.priority,
|
||||
connection_kind: policy.connection_kind,
|
||||
reservation_ttl_secs: session.read().await.account_overlap_timing().reservation_ttl_secs(),
|
||||
preacquired_provider_handle: None,
|
||||
started_generation: None,
|
||||
prepared_account: None,
|
||||
playback_kind: session.read().await.origin_playback_kind(),
|
||||
account_runtime: session.read().await.origin_account_runtime.clone(),
|
||||
};
|
||||
let manifest_commit_requirement = tuliprox_hls::api::hls_manifest_commit_requirement(
|
||||
session,
|
||||
tuliprox_hls::api::HlsSessionStoreOutcome::Reused,
|
||||
None,
|
||||
false,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
let request = hls_origin_refresh_request(
|
||||
&app_state,
|
||||
session,
|
||||
HlsPreparedOriginAccount { entry: entry.clone(), snapshot: snapshot.clone() },
|
||||
HlsOriginRefreshOptions {
|
||||
headers: origin_headers,
|
||||
origin_io,
|
||||
manifest_commit_requirement,
|
||||
acceptance_directive: HlsManifestAcceptanceDirective::none(),
|
||||
access_lease_id: None,
|
||||
rewrite_secret: app_state.get_encrypt_secret().to_vec(),
|
||||
now_ms: current_time_millis(),
|
||||
},
|
||||
);
|
||||
trigger_origin_refresh_sync(request).await;
|
||||
if session.read().await.origin_account_manifest_pending.is_some() {
|
||||
return Err(HlsBoundAccountAcquireErrorKind::Unavailable);
|
||||
}
|
||||
}
|
||||
Ok(HlsPreparedOriginAccount { entry, snapshot })
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tuliprox_hls::api::{HlsSession, HlsSessionKey};
|
||||
|
||||
#[test]
|
||||
fn rebind_publication_rejects_superseded_work_binding_and_gc() {
|
||||
let mut session = HlsSession::new(HlsSessionKey::new(1, "publication"), b"secret", 0);
|
||||
let expected =
|
||||
HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("expired"), &session.proxy_session_id, 0);
|
||||
session.replace_origin_account_binding(Some(expected.clone()));
|
||||
let replacement = HlsOriginAccountBinding::rebound(
|
||||
Arc::from("input"),
|
||||
Arc::from("replacement"),
|
||||
expected.session_owner.clone(),
|
||||
1,
|
||||
1,
|
||||
);
|
||||
session.invalidate_queued_origin_work();
|
||||
assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 0, replacement.clone()));
|
||||
assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name, expected.account_name);
|
||||
assert!(session.origin_account_manifest_pending.is_none());
|
||||
session.origin_account_binding.as_mut().unwrap().generation = 7;
|
||||
assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone()));
|
||||
assert_eq!(session.origin_account_binding.as_ref().unwrap().generation, 7);
|
||||
session.replace_origin_account_binding(Some(expected.clone()));
|
||||
session.mark_for_gc_removal();
|
||||
assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone()));
|
||||
session.clear_gc_removal_mark();
|
||||
assert!(publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement));
|
||||
assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "replacement");
|
||||
assert_eq!(session.activity.origin_work_generation, 2);
|
||||
assert_eq!(session.origin_account_manifest_pending, Some(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
use super::{AppState, HlsOriginIoContext, HlsSessionHandle};
|
||||
use axum::http::HeaderMap;
|
||||
use std::sync::Arc;
|
||||
use tuliprox_hls::api::{
|
||||
HlsAccessLeaseId, HlsManifestAcceptanceDirective, HlsManifestCommitRequirement, HlsPostRefreshRuntime,
|
||||
HlsPreparedOriginAccount, OriginRefreshRequest, RetryPolicy,
|
||||
};
|
||||
|
||||
pub(super) struct HlsOriginRefreshOptions {
|
||||
pub headers: HeaderMap,
|
||||
pub origin_io: HlsOriginIoContext,
|
||||
pub manifest_commit_requirement: HlsManifestCommitRequirement,
|
||||
pub acceptance_directive: HlsManifestAcceptanceDirective,
|
||||
pub access_lease_id: Option<HlsAccessLeaseId>,
|
||||
pub rewrite_secret: Vec<u8>,
|
||||
pub now_ms: u64,
|
||||
}
|
||||
|
||||
pub(super) fn hls_origin_refresh_request(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
prepared: HlsPreparedOriginAccount,
|
||||
options: HlsOriginRefreshOptions,
|
||||
) -> OriginRefreshRequest {
|
||||
let mut origin_io = options.origin_io;
|
||||
let origin_provider_session_headers = prepared.snapshot.provider_session_headers.clone();
|
||||
origin_io.prepared_account = Some(prepared.snapshot);
|
||||
OriginRefreshRequest {
|
||||
app_config: Arc::clone(&app_state.app_config),
|
||||
session: Arc::clone(session),
|
||||
origin_entry: prepared.entry,
|
||||
headers: options.headers,
|
||||
origin_provider_session_headers,
|
||||
disabled_headers: app_state.get_disabled_headers(),
|
||||
client: app_state.http_clients.default.load().as_ref().clone(),
|
||||
no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(),
|
||||
use_manual_redirects: app_state.should_use_manual_redirects(),
|
||||
segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()),
|
||||
hls_proxy: Arc::clone(&app_state.hls.proxy),
|
||||
segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()),
|
||||
segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()),
|
||||
map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()),
|
||||
origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(),
|
||||
manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(),
|
||||
strip: app_state.hls.proxy.strip().clone(),
|
||||
retry_policy: RetryPolicy::default(),
|
||||
reverse_proxy_rewrite_secret: options.rewrite_secret,
|
||||
transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(),
|
||||
manifest_commit_requirement: options.manifest_commit_requirement,
|
||||
fresh_manifest_requirement_generation: None,
|
||||
refresh_identity: None,
|
||||
acceptance_directive: options.acceptance_directive,
|
||||
access_lease_id: options.access_lease_id,
|
||||
now_ms: options.now_ms,
|
||||
origin_io: Some(origin_io),
|
||||
post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: app_state.hls_ctx().downgrade() }),
|
||||
}
|
||||
}
|
||||
@@ -186,7 +186,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_lease_allows_cached_segment(
|
||||
match &lease.playback_mode {
|
||||
HlsLeasePlaybackMode::Live => true,
|
||||
HlsLeasePlaybackMode::TerminalTail(plan) => plan.protected_base_proxy_seqs.contains(&proxy_seq),
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => false,
|
||||
HlsLeasePlaybackMode::TerminalCustomResponse(_)
|
||||
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
|
||||
| HlsLeasePlaybackMode::Ended => false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -94,7 +94,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_runtime_or_standalone_custom
|
||||
return fallback_status.into_response();
|
||||
};
|
||||
match &lease.playback_mode {
|
||||
HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => {
|
||||
HlsLeasePlaybackMode::TerminalTail(_)
|
||||
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
|
||||
| HlsLeasePlaybackMode::TerminalUnavailable { .. } => {
|
||||
if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) {
|
||||
return response;
|
||||
}
|
||||
@@ -196,7 +198,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_origin_runtime_resource_failure_re
|
||||
err: HlsOriginRuntimeAcquireError,
|
||||
) -> axum::response::Response {
|
||||
match err {
|
||||
HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => StatusCode::SERVICE_UNAVAILABLE.into_response(),
|
||||
HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => {
|
||||
hls_canonical_status_response(StatusCode::SERVICE_UNAVAILABLE)
|
||||
}
|
||||
HlsOriginRuntimeAcquireError::Fatal(status) => hls_canonical_status_response(status),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,10 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_hls_cache_stream_register
|
||||
access: &HlsAccessContext,
|
||||
session: &HlsSessionHandle,
|
||||
) -> Option<StreamInfo> {
|
||||
// Share the existing origin-publication gate: a registration cannot retain an old
|
||||
// account snapshot across a handover and then overwrite its projected provider.
|
||||
let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock);
|
||||
let _prepare_guard = prepare_lock.lock().await;
|
||||
let (proxy_session_id, origin_source, origin_account_binding) = {
|
||||
let session = session.read().await;
|
||||
if session.is_gc_marked_for_removal() {
|
||||
|
||||
@@ -145,9 +145,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_resource(
|
||||
match &lease.playback_mode {
|
||||
HlsLeasePlaybackMode::Live => serve_hls_live_transient_resource(context).await,
|
||||
HlsLeasePlaybackMode::TerminalTail(_) => serve_hls_terminal_key_resource(context, &lease.playback_mode).await,
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => {
|
||||
StatusCode::NOT_FOUND.into_response()
|
||||
}
|
||||
HlsLeasePlaybackMode::TerminalCustomResponse(_)
|
||||
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
|
||||
| HlsLeasePlaybackMode::Ended => StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,7 +226,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc
|
||||
};
|
||||
let resource = cache_resolution.resource;
|
||||
let origin_headers = cache_resolution.origin_headers;
|
||||
let origin_provider_session_headers = cache_resolution.origin_provider_session_headers;
|
||||
let cache_action = cache_resolution.action;
|
||||
|
||||
match cache_action {
|
||||
@@ -266,7 +265,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc
|
||||
resource,
|
||||
cache_action,
|
||||
origin_headers,
|
||||
origin_provider_session_headers,
|
||||
cache_duration_ms,
|
||||
})
|
||||
.await
|
||||
@@ -277,21 +275,14 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientPassthroughContext<'a>
|
||||
pub(in crate::api::endpoints::hls_api) resource: TransientResourceRef,
|
||||
pub(in crate::api::endpoints::hls_api) cache_action: HlsTransientObjectCacheAction,
|
||||
pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap,
|
||||
pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap,
|
||||
pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64,
|
||||
}
|
||||
|
||||
pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_resource(
|
||||
context: HlsTransientPassthroughContext<'_>,
|
||||
) -> axum::response::Response {
|
||||
let HlsTransientPassthroughContext {
|
||||
endpoint,
|
||||
resource,
|
||||
cache_action,
|
||||
origin_headers,
|
||||
origin_provider_session_headers,
|
||||
cache_duration_ms,
|
||||
} = context;
|
||||
let HlsTransientPassthroughContext { endpoint, resource, cache_action, origin_headers, cache_duration_ms } =
|
||||
context;
|
||||
if let HlsTransientObjectCacheAction::FetchAndCache(fetch_token) = cache_action {
|
||||
return fetch_and_cache_transient_origin_response(HlsTransientEndpointCacheFetchContext {
|
||||
app_state: endpoint.app_state,
|
||||
@@ -304,7 +295,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r
|
||||
resource_file: endpoint.resource_file,
|
||||
fetch_token: *fetch_token,
|
||||
origin_headers,
|
||||
origin_provider_session_headers,
|
||||
range_header: endpoint.range_header,
|
||||
cache_duration_ms,
|
||||
})
|
||||
@@ -321,7 +311,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r
|
||||
resource: &resource,
|
||||
resource_file: &endpoint.resource_file,
|
||||
origin_headers,
|
||||
origin_provider_session_headers,
|
||||
range_header: endpoint.range_header.clone(),
|
||||
policy: policy.clone(),
|
||||
})
|
||||
|
||||
@@ -229,6 +229,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_terminal_segment(
|
||||
}
|
||||
HlsLeasePlaybackMode::Live
|
||||
| HlsLeasePlaybackMode::TerminalTail(_)
|
||||
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
|
||||
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
|
||||
| HlsLeasePlaybackMode::Ended => None,
|
||||
},
|
||||
@@ -318,6 +319,8 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context(
|
||||
fingerprint: &Fingerprint,
|
||||
preacquired_provider_handle: Option<ProviderHandle>,
|
||||
) -> SegmentFetchContext {
|
||||
let preacquired_provider_handle = preacquired_provider_handle
|
||||
.map(|handle| tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle));
|
||||
let (headers, origin_provider_session_headers, origin_policy, reservation_ttl_secs) = {
|
||||
let session = session.read().await;
|
||||
(
|
||||
@@ -336,9 +339,12 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context(
|
||||
reservation_ttl_secs,
|
||||
preacquired_provider_handle: None,
|
||||
started_generation: None,
|
||||
prepared_account: None,
|
||||
playback_kind: session.read().await.origin_playback_kind(),
|
||||
account_runtime: session.read().await.origin_account_runtime.clone(),
|
||||
};
|
||||
if let Some(provider_handle) = preacquired_provider_handle {
|
||||
origin_io = origin_io.with_preacquired_provider_handle(provider_handle);
|
||||
origin_io = origin_io.with_managed_provider_handle(provider_handle);
|
||||
}
|
||||
SegmentFetchContext {
|
||||
session: Arc::clone(session),
|
||||
|
||||
@@ -35,7 +35,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointOriginFetchReq
|
||||
pub(in crate::api::endpoints::hls_api) resource: &'a TransientResourceRef,
|
||||
pub(in crate::api::endpoints::hls_api) resource_file: &'a TransientResourceFile,
|
||||
pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap,
|
||||
pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap,
|
||||
pub(in crate::api::endpoints::hls_api) range_header: Option<HeaderValue>,
|
||||
pub(in crate::api::endpoints::hls_api) policy: SegmentFetchPolicy,
|
||||
}
|
||||
@@ -54,14 +53,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_
|
||||
no_redirect_client: request.app_state.http_clients.no_redirect.load().as_ref().clone(),
|
||||
use_manual_redirects: request.app_state.should_use_manual_redirects(),
|
||||
};
|
||||
let log_identity = {
|
||||
let (log_identity, generation, provider_headers) = {
|
||||
let session = request.session.read().await;
|
||||
HlsLogIdentity::from_session(&session)
|
||||
(
|
||||
HlsLogIdentity::from_session(&session),
|
||||
session.activity.origin_work_generation,
|
||||
session.origin_provider_session_headers.clone(),
|
||||
)
|
||||
};
|
||||
let fetch_request = HlsTransientOriginFetchRequest {
|
||||
resolved_origin_uri: request.resource.resolved_origin_uri.clone(),
|
||||
origin_headers: request.origin_headers,
|
||||
origin_provider_session_headers: request.origin_provider_session_headers,
|
||||
origin_provider_session_headers: provider_headers,
|
||||
range_header: request.range_header,
|
||||
resource_file: request.resource_file.clone(),
|
||||
resource_kind: request.resource.kind,
|
||||
@@ -77,12 +80,19 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_
|
||||
request.fingerprint,
|
||||
request.headers,
|
||||
&runtime_prepare_error,
|
||||
HlsTransientOriginBindingSnapshot { resource: request.resource.clone(), generation },
|
||||
);
|
||||
let result = fetch_hls_transient_origin_response_with_attempt_prepare(fetch_request, prepare_attempt).await;
|
||||
let runtime_prepare_error = *runtime_prepare_error.lock().await;
|
||||
HlsTransientOriginFetchResult { result, runtime_prepare_error }
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(in crate::api::endpoints::hls_api) struct HlsTransientOriginBindingSnapshot {
|
||||
resource: TransientResourceRef,
|
||||
generation: u64,
|
||||
}
|
||||
|
||||
/// Builds the shared per-attempt prepare closure for transient origin fetches.
|
||||
/// Runtime acquire failures are captured in `runtime_prepare_error` and mapped
|
||||
/// to a provider-unavailable fetch error so the retry loop can proceed uniformly.
|
||||
@@ -93,6 +103,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure(
|
||||
fingerprint: &Fingerprint,
|
||||
headers: &HeaderMap,
|
||||
runtime_prepare_error: &Arc<tokio::sync::Mutex<Option<HlsOriginRuntimeAcquireError>>>,
|
||||
snapshot: HlsTransientOriginBindingSnapshot,
|
||||
) -> impl FnMut(
|
||||
HlsResourceFetchAttempt,
|
||||
) -> futures::future::BoxFuture<
|
||||
@@ -112,6 +123,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure(
|
||||
let fingerprint = fingerprint.clone();
|
||||
let headers = headers.clone();
|
||||
let runtime_prepare_error = Arc::clone(&runtime_prepare_error);
|
||||
let snapshot = snapshot.clone();
|
||||
async move {
|
||||
match prepare_hls_transient_origin_io_for_authorized_resource_work(
|
||||
&app_state,
|
||||
@@ -123,7 +135,17 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(guard) => Ok(guard),
|
||||
Ok(guard) => {
|
||||
let session = session.read().await;
|
||||
if session.activity.origin_work_generation != snapshot.generation
|
||||
|| !session.transient.origin_resource_is_fetchable(&snapshot.resource, current_time_millis())
|
||||
{
|
||||
return Err(HlsOriginResourceFetchError::ProviderUnavailable(
|
||||
HlsBoundAccountAcquireErrorKind::StoreRace,
|
||||
));
|
||||
}
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => {
|
||||
*runtime_prepare_error.lock().await = Some(err);
|
||||
Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Unavailable))
|
||||
@@ -146,7 +168,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointCacheFetchCont
|
||||
pub(in crate::api::endpoints::hls_api) resource_file: TransientResourceFile,
|
||||
pub(in crate::api::endpoints::hls_api) fetch_token: TransientObjectFetchToken,
|
||||
pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap,
|
||||
pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap,
|
||||
pub(in crate::api::endpoints::hls_api) range_header: Option<HeaderValue>,
|
||||
pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64,
|
||||
}
|
||||
@@ -298,14 +319,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin
|
||||
no_redirect_client: context.app_state.http_clients.no_redirect.load().as_ref().clone(),
|
||||
use_manual_redirects: context.app_state.should_use_manual_redirects(),
|
||||
};
|
||||
let log_identity = {
|
||||
let (log_identity, generation, provider_headers) = {
|
||||
let session = context.session.read().await;
|
||||
HlsLogIdentity::from_session(&session)
|
||||
(
|
||||
HlsLogIdentity::from_session(&session),
|
||||
session.activity.origin_work_generation,
|
||||
session.origin_provider_session_headers.clone(),
|
||||
)
|
||||
};
|
||||
let fetch_request = HlsTransientOriginFetchRequest {
|
||||
resolved_origin_uri: context.resource.resolved_origin_uri.clone(),
|
||||
origin_headers: context.origin_headers.clone(),
|
||||
origin_provider_session_headers: context.origin_provider_session_headers.clone(),
|
||||
origin_provider_session_headers: provider_headers,
|
||||
range_header: None,
|
||||
resource_file: context.resource_file.clone(),
|
||||
resource_kind: context.resource.kind,
|
||||
@@ -336,6 +361,7 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin
|
||||
context.fingerprint,
|
||||
context.headers,
|
||||
&runtime_prepare_error,
|
||||
HlsTransientOriginBindingSnapshot { resource: context.resource.clone(), generation },
|
||||
);
|
||||
let final_failure =
|
||||
match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(cache_fetch_request, prepare_attempt)
|
||||
|
||||
@@ -378,13 +378,37 @@ pub(super) fn hls_url_failover_provider_for_origin_context(
|
||||
}
|
||||
|
||||
pub(super) struct PreparedHlsOriginRuntime {
|
||||
pub(super) fetch_url: String,
|
||||
pub(super) runtime_provider_config: Option<Arc<RuntimeProviderConfig>>,
|
||||
pub(super) url_failover_provider: Option<Arc<ConfigProvider>>,
|
||||
pub(super) snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot,
|
||||
pub(super) preacquired_origin_account_handle: Option<tuliprox_session::ManagedProviderHandle>,
|
||||
}
|
||||
|
||||
impl PreparedHlsOriginRuntime {
|
||||
pub(super) fn published(
|
||||
allocation: HlsOriginRuntimeAllocation,
|
||||
snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot,
|
||||
) -> Self {
|
||||
Self {
|
||||
fetch_url: allocation.fetch_url,
|
||||
runtime_provider_config: allocation.runtime_provider_config,
|
||||
url_failover_provider: allocation.url_failover_provider,
|
||||
snapshot,
|
||||
preacquired_origin_account_handle: allocation.preacquired_origin_account_handle,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct HlsOriginRuntimeAllocation {
|
||||
pub(super) fetch_url: String,
|
||||
pub(super) runtime_provider_config: Option<Arc<RuntimeProviderConfig>>,
|
||||
// URL failover comes from source.yml provider:// resolution. Origin-account
|
||||
// binding/handles are runtime account reservations and must stay separate.
|
||||
pub(super) url_failover_provider: Option<Arc<ConfigProvider>>,
|
||||
pub(super) origin_account_binding_to_store: Option<HlsOriginAccountBinding>,
|
||||
pub(super) preacquired_origin_account_handle: Option<ProviderHandle>,
|
||||
pub(super) binding_to_publish: Option<HlsOriginAccountBinding>,
|
||||
pub(super) prepared_snapshot: Option<tuliprox_hls::api::HlsOriginAccountSnapshot>,
|
||||
pub(super) preacquired_origin_account_handle: Option<tuliprox_session::ManagedProviderHandle>,
|
||||
}
|
||||
|
||||
pub(super) fn effective_hls_url_failover_provider_for_fetch_url(
|
||||
@@ -626,194 +650,8 @@ pub(super) fn hls_soft_overlap_delay_ms(
|
||||
///
|
||||
/// Archive-backed sessions must keep catchup semantics instead of plain live HLS;
|
||||
/// live sessions keep the live HLS kind.
|
||||
async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind {
|
||||
if session.read().await.key.archive_reference.is_some() {
|
||||
PlaybackKind::Catchup
|
||||
} else {
|
||||
PlaybackKind::LiveHls
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub(super) async fn prepare_hls_origin_runtime(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
input: &ConfigInput,
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
proxy_session_id: &ProxySessionId,
|
||||
fingerprint: &Fingerprint,
|
||||
connection_kind: crate::api::model::ConnectionKind,
|
||||
priority: i8,
|
||||
work_kind: HlsOriginWorkKind,
|
||||
work_class: HlsOriginWorkClass,
|
||||
now_ms: u64,
|
||||
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
|
||||
let playback_kind = hls_session_origin_playback_kind(session).await;
|
||||
promote_elapsed_hls_account_overlaps(app_state, now_ms).await;
|
||||
detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await;
|
||||
reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await;
|
||||
let existing_binding = session.read().await.origin_account_binding.clone();
|
||||
let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached);
|
||||
let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref());
|
||||
if reacquire_detached_binding {
|
||||
log_hls_origin_binding_reacquire_started(session, work_kind).await;
|
||||
}
|
||||
if let Some(binding) = existing_binding {
|
||||
if binding.is_active() {
|
||||
match hls_origin_account_status(&app_state.hls_ctx(), &binding) {
|
||||
stale_status @ (HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired) => {
|
||||
return rebind_hls_origin_account(
|
||||
app_state,
|
||||
session,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
&binding,
|
||||
stale_status,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
HlsOriginAccountStatus::Known => {
|
||||
return Ok(prepared_hls_origin_runtime_for_known_binding(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
&binding,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match prepare_hls_origin_runtime_with_new_account(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
false,
|
||||
work_kind,
|
||||
work_class,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => {
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)),
|
||||
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
|
||||
}
|
||||
|
||||
if work_class.allows_speculative_overlap() {
|
||||
if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime(
|
||||
app_state,
|
||||
session,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
|
||||
if let Ok(prepared) = prepare_hls_speculative_origin_runtime(
|
||||
app_state,
|
||||
session,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
} else {
|
||||
debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value());
|
||||
}
|
||||
|
||||
if work_class.allows_grace() {
|
||||
match prepare_hls_origin_runtime_with_new_account(
|
||||
app_state,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
proxy_session_id,
|
||||
fingerprint,
|
||||
connection_kind,
|
||||
priority,
|
||||
true,
|
||||
work_kind,
|
||||
work_class,
|
||||
playback_kind,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => {
|
||||
if reacquire_detached_binding {
|
||||
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
|
||||
log_hls_origin_binding_reacquired(session, binding).await;
|
||||
}
|
||||
}
|
||||
return Ok(prepared);
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(status))
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
"HLS origin account grace skipped: work_class={} reason=background-origin-work",
|
||||
work_class.as_log_value()
|
||||
);
|
||||
}
|
||||
|
||||
if reacquire_detached_binding {
|
||||
log_hls_origin_binding_reacquire_failed(session, "no-account-available").await;
|
||||
}
|
||||
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason })
|
||||
pub(super) async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind {
|
||||
session.read().await.origin_playback_kind()
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -831,7 +669,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
|
||||
work_class: HlsOriginWorkClass,
|
||||
playback_kind: PlaybackKind,
|
||||
now_ms: u64,
|
||||
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
|
||||
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
|
||||
let session_owner = build_hls_origin_session_owner(proxy_session_id);
|
||||
let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session(
|
||||
&input.name,
|
||||
@@ -852,8 +690,13 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
|
||||
});
|
||||
};
|
||||
|
||||
let managed_handle =
|
||||
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
|
||||
let Some(provider_handle) = managed_handle.handle() else {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
@@ -866,7 +709,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
|
||||
)
|
||||
.await
|
||||
else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
@@ -876,7 +719,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
|
||||
&provider_handle.allocation,
|
||||
now_ms,
|
||||
) else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
@@ -896,7 +739,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
|
||||
grace_state
|
||||
);
|
||||
|
||||
Ok(PreparedHlsOriginRuntime {
|
||||
Ok(HlsOriginRuntimeAllocation {
|
||||
url_failover_provider: hls_url_failover_provider_for_origin_context(
|
||||
input,
|
||||
raw_request_url,
|
||||
@@ -905,8 +748,9 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
|
||||
),
|
||||
runtime_provider_config: Some(provider_config),
|
||||
fetch_url,
|
||||
origin_account_binding_to_store: Some(binding),
|
||||
preacquired_origin_account_handle: Some(provider_handle),
|
||||
binding_to_publish: Some(binding),
|
||||
prepared_snapshot: None,
|
||||
preacquired_origin_account_handle: Some(managed_handle),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -916,7 +760,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding(
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
binding: &HlsOriginAccountBinding,
|
||||
) -> PreparedHlsOriginRuntime {
|
||||
) -> HlsOriginRuntimeAllocation {
|
||||
let runtime_provider_config = app_state.active_provider.find_provider_config(&binding.account_name);
|
||||
let fetch_url = resolve_hls_origin_fetch_url(
|
||||
&app_state.app_config,
|
||||
@@ -928,7 +772,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding(
|
||||
.await
|
||||
.unwrap_or_else(|| session_entry_url.to_string());
|
||||
|
||||
PreparedHlsOriginRuntime {
|
||||
HlsOriginRuntimeAllocation {
|
||||
runtime_provider_config,
|
||||
url_failover_provider: hls_url_failover_provider_for_origin_context(
|
||||
input,
|
||||
@@ -937,7 +781,8 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding(
|
||||
&fetch_url,
|
||||
),
|
||||
fetch_url,
|
||||
origin_account_binding_to_store: None,
|
||||
binding_to_publish: None,
|
||||
prepared_snapshot: None,
|
||||
preacquired_origin_account_handle: None,
|
||||
}
|
||||
}
|
||||
@@ -995,14 +840,15 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A
|
||||
timing.target_duration_ms
|
||||
);
|
||||
let starting_window_ms = tuliprox_session::STARTING_LEASE_TTL_SECS.saturating_mul(1000);
|
||||
let media_window_expired = session_guard.activity.last_delivered_media_at_ms.map_or_else(
|
||||
|| now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms),
|
||||
|last| {
|
||||
now_ms
|
||||
> last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms)
|
||||
},
|
||||
);
|
||||
if !media_window_expired
|
||||
// A retained session can contain delivery from an earlier playback.
|
||||
// Give each newly published binding its own bounded start window;
|
||||
// healthy preparation leaves pinned_at_ms unchanged.
|
||||
let starting_window_expired = now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms);
|
||||
let media_window_expired = session_guard.activity.last_delivered_media_at_ms.is_none_or(|last| {
|
||||
now_ms > last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms)
|
||||
});
|
||||
if !starting_window_expired
|
||||
|| !media_window_expired
|
||||
|| session_guard.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0
|
||||
{
|
||||
continue;
|
||||
@@ -1033,7 +879,6 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
app_state: &Arc<AppState>,
|
||||
new_session: &HlsSessionHandle,
|
||||
input: &ConfigInput,
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
@@ -1043,7 +888,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
priority: i8,
|
||||
playback_kind: PlaybackKind,
|
||||
now_ms: u64,
|
||||
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
|
||||
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
|
||||
let request_policy = HlsEffectiveOriginAcquirePolicy::new(connection_kind, priority, now_ms);
|
||||
let Some(candidate) =
|
||||
find_hls_origin_policy_preempt_candidate(app_state, input, proxy_session_id, request_policy, now_ms).await
|
||||
@@ -1087,8 +932,13 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
debug!("HLS origin policy preemption denied: reason=exact-acquire-failed");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
let managed_handle =
|
||||
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
|
||||
let Some(provider_handle) = managed_handle.handle() else {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
|
||||
debug!("HLS origin policy preemption denied: reason=missing-provider-config");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
@@ -1102,7 +952,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
)
|
||||
.await
|
||||
else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
|
||||
debug!("HLS origin policy preemption denied: reason=invalid-origin-url");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
@@ -1113,7 +963,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
&provider_handle.allocation,
|
||||
now_ms,
|
||||
) else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
|
||||
debug!("HLS origin policy preemption denied: reason=invalid-allocation");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
@@ -1140,16 +990,12 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
}
|
||||
}
|
||||
if !detached_victim {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
|
||||
debug!("HLS origin policy preemption denied: reason=stale-candidate");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
|
||||
{
|
||||
let mut session_guard = new_session.write().await;
|
||||
session_guard.replace_origin_account_binding(Some(binding.clone()));
|
||||
}
|
||||
debug!(
|
||||
"HLS origin policy preempted: account={} victim_proxy_session={} winner_proxy_session={} victim_kind={:?} victim_priority={} request_kind={:?} request_priority={}",
|
||||
sanitize_sensitive_info(candidate.account_name.as_ref()),
|
||||
@@ -1167,7 +1013,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
HlsOriginAccountDetachedReason::PreemptedByHigherPriority.as_log_reason()
|
||||
);
|
||||
|
||||
Ok(PreparedHlsOriginRuntime {
|
||||
Ok(HlsOriginRuntimeAllocation {
|
||||
url_failover_provider: hls_url_failover_provider_for_origin_context(
|
||||
input,
|
||||
raw_request_url,
|
||||
@@ -1176,8 +1022,9 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
|
||||
),
|
||||
runtime_provider_config: Some(provider_config),
|
||||
fetch_url,
|
||||
origin_account_binding_to_store: Some(binding),
|
||||
preacquired_origin_account_handle: Some(provider_handle),
|
||||
binding_to_publish: Some(binding),
|
||||
prepared_snapshot: None,
|
||||
preacquired_origin_account_handle: Some(managed_handle),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1283,7 +1130,6 @@ pub(super) fn hls_origin_policy_preempt_candidate_is_better(
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn prepare_hls_speculative_origin_runtime(
|
||||
app_state: &Arc<AppState>,
|
||||
new_session: &HlsSessionHandle,
|
||||
input: &ConfigInput,
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
@@ -1293,7 +1139,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
|
||||
priority: i8,
|
||||
playback_kind: PlaybackKind,
|
||||
now_ms: u64,
|
||||
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
|
||||
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
|
||||
let Some(candidate) = find_hls_account_overlap_candidate(app_state, input, proxy_session_id, now_ms).await else {
|
||||
debug!("HLS account overlap denied: reason=no-soft-active-candidate");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
@@ -1316,8 +1162,13 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
|
||||
debug!("HLS account overlap denied: reason=speculative-acquire-failed");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
let managed_handle =
|
||||
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
|
||||
let Some(provider_handle) = managed_handle.handle() else {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
|
||||
debug!("HLS account overlap denied: reason=missing-provider-config");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
@@ -1331,7 +1182,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
|
||||
)
|
||||
.await
|
||||
else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
drop(managed_handle);
|
||||
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
|
||||
debug!("HLS account overlap denied: reason=invalid-origin-url");
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
@@ -1344,10 +1195,6 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
|
||||
candidate.reclaim_until_ms,
|
||||
now_ms,
|
||||
);
|
||||
{
|
||||
let mut session_guard = new_session.write().await;
|
||||
session_guard.replace_origin_account_binding(Some(binding.clone()));
|
||||
}
|
||||
debug!(
|
||||
"HLS account overlap granted: account={} victim_proxy_session={} winner_proxy_session={} reclaim_until_ms={} eligible_after_ms={} delay_ms={} tuliprox_target_user_connections={} origin_input_account_connections={}",
|
||||
sanitize_sensitive_info(candidate.account_name.as_ref()),
|
||||
@@ -1359,7 +1206,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
|
||||
candidate.tuliprox_target_user_connection_capacity,
|
||||
candidate.origin_input_account_connection_capacity
|
||||
);
|
||||
Ok(PreparedHlsOriginRuntime {
|
||||
Ok(HlsOriginRuntimeAllocation {
|
||||
url_failover_provider: hls_url_failover_provider_for_origin_context(
|
||||
input,
|
||||
raw_request_url,
|
||||
@@ -1368,8 +1215,9 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
|
||||
),
|
||||
runtime_provider_config: Some(provider_config),
|
||||
fetch_url,
|
||||
origin_account_binding_to_store: Some(binding),
|
||||
preacquired_origin_account_handle: Some(provider_handle),
|
||||
binding_to_publish: Some(binding),
|
||||
prepared_snapshot: None,
|
||||
preacquired_origin_account_handle: Some(managed_handle),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1650,154 +1498,6 @@ pub(super) async fn promote_elapsed_hls_account_overlaps(app_state: &Arc<AppStat
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[allow(clippy::too_many_lines)]
|
||||
pub(super) async fn rebind_hls_origin_account(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
input: &ConfigInput,
|
||||
raw_request_url: &str,
|
||||
session_entry_url: &str,
|
||||
stale_binding: &HlsOriginAccountBinding,
|
||||
stale_status: HlsOriginAccountStatus,
|
||||
fingerprint: &Fingerprint,
|
||||
connection_kind: crate::api::model::ConnectionKind,
|
||||
priority: i8,
|
||||
playback_kind: PlaybackKind,
|
||||
now_ms: u64,
|
||||
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
|
||||
{
|
||||
let mut session_guard = session.write().await;
|
||||
if !session_guard.origin_account_rebind.is_allowed_now(now_ms) {
|
||||
debug!(
|
||||
"HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000",
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id),
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref())
|
||||
);
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms);
|
||||
}
|
||||
|
||||
let safe_proxy_session = {
|
||||
let session_guard = session.read().await;
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id)
|
||||
};
|
||||
debug!(
|
||||
"HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}",
|
||||
safe_proxy_session,
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref())
|
||||
);
|
||||
app_state.active_provider.clear_identified_provider_reservation(
|
||||
&stale_binding.session_owner,
|
||||
&stale_binding.account_name,
|
||||
stale_binding.provider_binding_tag,
|
||||
);
|
||||
{
|
||||
let mut session_guard = session.write().await;
|
||||
if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| {
|
||||
binding.account_name == stale_binding.account_name
|
||||
&& binding.session_owner == stale_binding.session_owner
|
||||
&& binding.provider_binding_tag == stale_binding.provider_binding_tag
|
||||
}) {
|
||||
binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms);
|
||||
debug!(
|
||||
"HLS origin binding detached: proxy_session={} account={} reason={}",
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id),
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
|
||||
HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason()
|
||||
);
|
||||
}
|
||||
session_guard.invalidate_queued_origin_work();
|
||||
}
|
||||
|
||||
let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session(
|
||||
&input.name,
|
||||
&fingerprint.addr,
|
||||
false,
|
||||
priority,
|
||||
connection_kind,
|
||||
Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)),
|
||||
) else {
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
|
||||
reason: hls_no_account_reason_for_binding(Some(stale_binding)),
|
||||
});
|
||||
};
|
||||
|
||||
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
let Some(fetch_url) = resolve_hls_origin_fetch_url(
|
||||
&app_state.app_config,
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
Some(&provider_config),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
let Some(new_account_name) = provider_handle.allocation.get_provider_name() else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await;
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
|
||||
let new_binding = HlsOriginAccountBinding::rebound(
|
||||
Arc::clone(&input.name),
|
||||
new_account_name,
|
||||
stale_binding.session_owner.clone(),
|
||||
stale_binding.generation.saturating_add(1),
|
||||
now_ms,
|
||||
);
|
||||
{
|
||||
let mut session_guard = session.write().await;
|
||||
session_guard.replace_origin_account_binding(Some(new_binding.clone()));
|
||||
session_guard.origin_account_rebind.mark_success();
|
||||
}
|
||||
debug!(
|
||||
"HLS origin account rebound: old_account={} new_account={}",
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
|
||||
sanitize_sensitive_info(new_binding.account_name.as_ref())
|
||||
);
|
||||
|
||||
Ok(PreparedHlsOriginRuntime {
|
||||
url_failover_provider: hls_url_failover_provider_for_origin_context(
|
||||
input,
|
||||
raw_request_url,
|
||||
session_entry_url,
|
||||
&fetch_url,
|
||||
),
|
||||
runtime_provider_config: Some(provider_config),
|
||||
fetch_url,
|
||||
origin_account_binding_to_store: None,
|
||||
preacquired_origin_account_handle: Some(provider_handle),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn mark_hls_origin_rebind_failed(
|
||||
session: &HlsSessionHandle,
|
||||
stale_binding: &HlsOriginAccountBinding,
|
||||
now_ms: u64,
|
||||
reason: &str,
|
||||
) {
|
||||
let mut session_guard = session.write().await;
|
||||
session_guard.origin_account_rebind.mark_failed(now_ms);
|
||||
debug!(
|
||||
"HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000",
|
||||
safe_proxy_session_id(&session_guard.proxy_session_id),
|
||||
sanitize_sensitive_info(stale_binding.account_name.as_ref())
|
||||
);
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn prepare_hls_cache_user_session(
|
||||
app_state: &Arc<AppState>,
|
||||
@@ -1995,6 +1695,7 @@ pub(super) async fn hls_segment_request_requires_origin_work(
|
||||
&& entry.origin_fetch_ref.is_some()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn hls_origin_binding_needs_reacquire(session: &HlsSessionHandle) -> bool {
|
||||
let session = session.read().await;
|
||||
session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached)
|
||||
@@ -2008,7 +1709,7 @@ pub(super) fn hls_transient_origin_binding_requires_runtime_prepare(
|
||||
|| (binding.is_active()
|
||||
&& matches!(
|
||||
hls_origin_account_status(hls_ctx, binding),
|
||||
HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired
|
||||
HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired | HlsOriginAccountStatus::Blocked
|
||||
))
|
||||
}
|
||||
|
||||
@@ -2021,11 +1722,22 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work(
|
||||
work_kind: HlsOriginWorkKind,
|
||||
now_ms: u64,
|
||||
) -> Result<Option<ProviderHandle>, HlsOriginRuntimeAcquireError> {
|
||||
if !hls_origin_binding_needs_reacquire(session).await {
|
||||
let runtime = session.read().await.origin_account_runtime.clone();
|
||||
if let Some(runtime) = runtime {
|
||||
runtime
|
||||
.prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand)
|
||||
.await
|
||||
.map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?;
|
||||
return Ok(None);
|
||||
}
|
||||
if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
if !session
|
||||
.read()
|
||||
.await
|
||||
.origin_account_binding
|
||||
.as_ref()
|
||||
.is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&app_state.hls_ctx(), binding))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers)
|
||||
.await
|
||||
@@ -2047,13 +1759,17 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work(
|
||||
now_ms,
|
||||
)
|
||||
.await?;
|
||||
if let Some(binding) = prepared_origin.origin_account_binding_to_store {
|
||||
session.write().await.replace_origin_account_binding(Some(binding));
|
||||
drop(prepared_origin);
|
||||
let runtime = session.read().await.origin_account_runtime.clone();
|
||||
if let Some(runtime) = runtime {
|
||||
runtime
|
||||
.prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand)
|
||||
.await
|
||||
.map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?;
|
||||
}
|
||||
Ok(prepared_origin.preacquired_origin_account_handle)
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
@@ -2062,120 +1778,55 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work
|
||||
req_headers: &HeaderMap,
|
||||
now_ms: u64,
|
||||
) -> Result<Option<HlsTransientOriginIoGuard>, HlsOriginRuntimeAcquireError> {
|
||||
let hls_ctx = app_state.hls_ctx();
|
||||
let existing_binding = session.read().await.origin_account_binding.clone();
|
||||
let origin_policy = hls_effective_origin_acquire_policy(session).await;
|
||||
let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(session).await;
|
||||
if let Some(binding) = existing_binding.as_ref().filter(|binding| binding.is_active()) {
|
||||
match hls_origin_account_status(&hls_ctx, binding) {
|
||||
HlsOriginAccountStatus::Known => {
|
||||
let origin_io = HlsOriginIoContext {
|
||||
ctx: hls_ctx.clone(),
|
||||
client_addr: fingerprint.addr,
|
||||
allow_grace: HlsOriginWorkClass::Demand.allows_grace(),
|
||||
priority: origin_policy.priority,
|
||||
connection_kind: origin_policy.connection_kind,
|
||||
reservation_ttl_secs,
|
||||
preacquired_provider_handle: None,
|
||||
started_generation: None,
|
||||
};
|
||||
if let Ok(lease_guard) = begin_hls_origin_account_io_bounded(
|
||||
&origin_io,
|
||||
session,
|
||||
binding,
|
||||
hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let (started_generation, active_origin_work_count) = {
|
||||
let mut session = session.write().await;
|
||||
let count = Arc::clone(&session.activity.active_origin_work_count);
|
||||
(session.start_origin_work(), count)
|
||||
};
|
||||
return Ok(Some(HlsTransientOriginIoGuard::new(
|
||||
Arc::clone(session),
|
||||
active_origin_work_count,
|
||||
origin_io,
|
||||
lease_guard,
|
||||
started_generation,
|
||||
)));
|
||||
}
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !existing_binding
|
||||
.as_ref()
|
||||
.is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, binding))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers)
|
||||
.await
|
||||
.map_err(HlsOriginRuntimeAcquireError::Fatal)?;
|
||||
let proxy_session_id = session.read().await.proxy_session_id.clone();
|
||||
let prepared_origin = prepare_hls_origin_runtime(
|
||||
prepare_hls_origin_binding_for_authorized_resource_work(
|
||||
app_state,
|
||||
session,
|
||||
&request_context.input,
|
||||
&request_context.hls_url,
|
||||
request_context.session_entry_url.as_str(),
|
||||
&proxy_session_id,
|
||||
access_context,
|
||||
fingerprint,
|
||||
origin_policy.connection_kind,
|
||||
origin_policy.priority,
|
||||
req_headers,
|
||||
HlsOriginWorkKind::Resource,
|
||||
HlsOriginWorkClass::Demand,
|
||||
now_ms,
|
||||
)
|
||||
.await?;
|
||||
if let Some(binding) = prepared_origin.origin_account_binding_to_store {
|
||||
session.write().await.replace_origin_account_binding(Some(binding));
|
||||
}
|
||||
let Some(provider_handle) = prepared_origin.preacquired_origin_account_handle else {
|
||||
let Some(binding) = session.read().await.origin_account_binding.clone() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(binding) = session.read().await.origin_account_binding.clone().filter(HlsOriginAccountBinding::is_active)
|
||||
else {
|
||||
app_state.connection_manager.release_provider_handle(Some(provider_handle));
|
||||
if binding.is_detached() {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
}
|
||||
let policy = hls_effective_origin_acquire_policy(session).await;
|
||||
let origin_io = HlsOriginIoContext {
|
||||
ctx: hls_ctx,
|
||||
ctx: app_state.hls_ctx(),
|
||||
client_addr: fingerprint.addr,
|
||||
allow_grace: HlsOriginWorkClass::Demand.allows_grace(),
|
||||
priority: origin_policy.priority,
|
||||
connection_kind: origin_policy.connection_kind,
|
||||
reservation_ttl_secs,
|
||||
priority: policy.priority,
|
||||
connection_kind: policy.connection_kind,
|
||||
reservation_ttl_secs: hls_origin_account_reservation_ttl_secs_for_session(session).await,
|
||||
preacquired_provider_handle: None,
|
||||
started_generation: None,
|
||||
}
|
||||
.with_preacquired_provider_handle(provider_handle);
|
||||
let Ok(lease_guard) = begin_hls_origin_account_io_bounded(
|
||||
prepared_account: None,
|
||||
playback_kind: session.read().await.origin_playback_kind(),
|
||||
account_runtime: session.read().await.origin_account_runtime.clone(),
|
||||
};
|
||||
let lease_guard = begin_hls_origin_account_io_bounded(
|
||||
&origin_io,
|
||||
session,
|
||||
&binding,
|
||||
hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
};
|
||||
let (started_generation, active_origin_work_count) = {
|
||||
.map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?;
|
||||
let (generation, active_count) = {
|
||||
let mut session = session.write().await;
|
||||
if !session.origin_account_binding.as_ref().is_some_and(|current| {
|
||||
current.generation == binding.generation
|
||||
&& current.account_name == binding.account_name
|
||||
&& current.is_active()
|
||||
}) {
|
||||
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
let count = Arc::clone(&session.activity.active_origin_work_count);
|
||||
(session.start_origin_work(), count)
|
||||
};
|
||||
Ok(Some(HlsTransientOriginIoGuard::new(
|
||||
Arc::clone(session),
|
||||
active_origin_work_count,
|
||||
origin_io,
|
||||
lease_guard,
|
||||
started_generation,
|
||||
)))
|
||||
Ok(Some(HlsTransientOriginIoGuard::new(Arc::clone(session), active_count, origin_io, lease_guard, generation)))
|
||||
}
|
||||
|
||||
@@ -78,15 +78,20 @@ use tuliprox_hls::api::{
|
||||
HlsCachedManifestOptions, HlsLeaseManifestSnapshotInput, LiveHlsOriginEntry,
|
||||
};
|
||||
|
||||
mod origin_account_rebind;
|
||||
mod owner_token;
|
||||
mod reduced_retry;
|
||||
mod same_account_resume;
|
||||
|
||||
mod archive;
|
||||
mod bandwidth;
|
||||
mod encrypted_media;
|
||||
mod entry_identity;
|
||||
mod headers;
|
||||
mod late_custom_response;
|
||||
mod lease_lifecycle;
|
||||
mod manifest_entry_freshness;
|
||||
mod manifest_short_window;
|
||||
mod manifest_startup;
|
||||
mod observability;
|
||||
mod owner_handoff;
|
||||
|
||||
@@ -163,7 +163,7 @@ async fn hls_cache_entry_returns_master_playlist_without_origin_refresh_or_sessi
|
||||
.is_some());
|
||||
assert!(app_state.hls.proxy.sessions().get_by_key(&session_key).await.is_none());
|
||||
assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0);
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -398,7 +398,7 @@ async fn legacy_hls_token_route_renders_channel_unavailable_inline_when_target_h
|
||||
assert!(!response.headers().contains_key(header::LOCATION));
|
||||
assert_eq!(response.headers()[header::CONTENT_TYPE], "application/vnd.apple.mpegurl");
|
||||
assert!(app_state.hls.proxy.access_leases().read().await.is_empty());
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -424,7 +424,7 @@ async fn legacy_hls_token_route_with_invalid_token_returns_bad_request_when_targ
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert!(app_state.hls.proxy.access_leases().read().await.is_empty());
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -545,7 +545,7 @@ async fn hls_entry_origin_reservation_blocks_foreign_sessions_only_after_media_c
|
||||
|
||||
assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8");
|
||||
assert!(reservation.selected_provider_config.is_some());
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
app_state.connection_manager.release_provider_handle(reservation.provider_handle);
|
||||
|
||||
// A manifest-only entry reservation is still unconfirmed, so it must not block a
|
||||
@@ -1053,7 +1053,7 @@ async fn hls_entry_for_input_in_maintenance_opens_no_session() {
|
||||
// No maintenance clip is configured here, so the configured fallback status answers.
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(app_state.hls.proxy.access_leases().read().await.is_empty(), "no lease for a maintenance input");
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_serves_canonical_manifest_and_immutable_media_without_origin_requests() {
|
||||
let fixture = terminal_tail::publication_late_fixture().await;
|
||||
let now_ms = super::super::current_time_millis();
|
||||
let lease = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
|
||||
.await
|
||||
.expect("published live lease");
|
||||
let snapshot = lease.last_manifest_snapshot.expect("published snapshot");
|
||||
{
|
||||
let mut session = fixture.session.write().await;
|
||||
session.segments.clear();
|
||||
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
|
||||
}
|
||||
let requests_before = fixture.origin.manifest_request_count();
|
||||
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(!response.headers().contains_key(header::LOCATION));
|
||||
assert!(!response.headers().contains_key(header::RETRY_AFTER));
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest");
|
||||
assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", snapshot.last_proxy_seq + 1)));
|
||||
assert!(body.contains(&format!("#EXT-X-TARGETDURATION:{}\n", snapshot.target_duration_ms / 1_000)));
|
||||
assert!(body.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n"));
|
||||
assert!(body.ends_with("#EXT-X-ENDLIST\n"));
|
||||
assert!(!body.contains("/hls/shared/live/"));
|
||||
let media_uris = body.lines().filter(|line| line.contains("/cvs/hls/")).collect::<Vec<_>>();
|
||||
assert_eq!(media_uris.len(), 12);
|
||||
let (first, second) = tokio::join!(
|
||||
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None),
|
||||
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None),
|
||||
);
|
||||
for response in [first, second] {
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response_body(response).await, body.as_bytes());
|
||||
}
|
||||
let media = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await;
|
||||
assert_eq!(media.status(), StatusCode::OK);
|
||||
assert_eq!(media.headers()[header::CONTENT_TYPE], "video/mp2t");
|
||||
let bytes = response_body(media).await;
|
||||
assert!(!bytes.is_empty());
|
||||
let range = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], Some("bytes=0-187")).await;
|
||||
assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT);
|
||||
assert_eq!(response_body(range).await, bytes.slice(..188));
|
||||
let router =
|
||||
crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(Arc::clone(&fixture.app_state));
|
||||
let head = router
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::HEAD)
|
||||
.uri(media_uris[0])
|
||||
.extension(ConnectInfo(test_addr()))
|
||||
.body(Body::empty())
|
||||
.expect("HEAD request"),
|
||||
)
|
||||
.await
|
||||
.expect("HEAD response");
|
||||
assert_eq!(head.status(), StatusCode::OK);
|
||||
assert_eq!(head.headers()[header::CONTENT_LENGTH], bytes.len().to_string());
|
||||
assert!(response_body(head).await.is_empty());
|
||||
fixture.app_state.app_config.custom_stream_response.store(None);
|
||||
let replay = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await;
|
||||
assert_eq!(replay.status(), StatusCode::OK);
|
||||
assert_eq!(response_body(replay).await, bytes);
|
||||
let wrong_lease = media_uris[0].replace(&fixture.access_lease_id.0, "unknown-lease");
|
||||
assert_eq!(
|
||||
custom_media_response(Arc::clone(&fixture.app_state), &wrong_lease, None).await.status(),
|
||||
StatusCode::NOT_FOUND
|
||||
);
|
||||
let wrong_index = media_uris[0].replace("/0.ts", "/12.ts");
|
||||
assert_eq!(
|
||||
custom_media_response(Arc::clone(&fixture.app_state), &wrong_index, None).await.status(),
|
||||
StatusCode::NOT_FOUND
|
||||
);
|
||||
let late = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.access_lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
super::super::current_time_millis(),
|
||||
)
|
||||
.await
|
||||
.expect("terminal lease");
|
||||
assert!(matches!(late.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
|
||||
assert_eq!(late.playback_cursor, lease.playback_cursor);
|
||||
assert!(!fixture.session.read().await.has_terminal_tail_protections());
|
||||
assert_eq!(fixture.origin.manifest_request_count(), requests_before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_without_configured_asset_retains_error_response() {
|
||||
let fixture = terminal_tail::publication_late_fixture().await;
|
||||
fixture.app_state.app_config.custom_stream_response.store(None);
|
||||
{
|
||||
let mut session = fixture.session.write().await;
|
||||
session.segments.clear();
|
||||
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
|
||||
}
|
||||
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(response_body(response).await.is_empty());
|
||||
let lease = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.access_lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
super::super::current_time_millis(),
|
||||
)
|
||||
.await
|
||||
.expect("terminal unavailable lease");
|
||||
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_media_get_head_and_range_renew_lease_without_manifest_polls() {
|
||||
let (fixture, media_uri) = committed_late_http_fixture().await;
|
||||
for (method, range, status) in [
|
||||
(Method::GET, None, StatusCode::OK),
|
||||
(Method::HEAD, None, StatusCode::OK),
|
||||
(Method::GET, Some("bytes=0-187"), StatusCode::PARTIAL_CONTENT),
|
||||
] {
|
||||
let now_ms = super::super::current_time_millis();
|
||||
let previous_access_ms = now_ms.saturating_sub(50_000);
|
||||
assert!(fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.activate_access_lease(
|
||||
&fixture.access_lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
previous_access_ms,
|
||||
HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 },
|
||||
)
|
||||
.await
|
||||
.is_activated());
|
||||
let before = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
|
||||
.await
|
||||
.expect("valid late lease");
|
||||
let response =
|
||||
custom_media_response_with_method(Arc::clone(&fixture.app_state), &media_uri, range, method).await;
|
||||
assert_eq!(response.status(), status);
|
||||
let after = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.access_lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
super::super::current_time_millis(),
|
||||
)
|
||||
.await
|
||||
.expect("renewed late lease");
|
||||
assert!(after.last_seen_at_ms >= now_ms);
|
||||
assert!(after.valid_until_ms > before.valid_until_ms);
|
||||
assert!(after.active_until_ms > before.active_until_ms);
|
||||
assert_eq!(after.playback_mode, before.playback_mode);
|
||||
assert_eq!(after.playback_cursor, before.playback_cursor);
|
||||
assert_eq!(after.admission_generation, before.admission_generation);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease() {
|
||||
let (fixture, media_uri) = committed_late_http_fixture().await;
|
||||
let now_ms = super::super::current_time_millis();
|
||||
assert!(fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.activate_access_lease(
|
||||
&fixture.access_lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
now_ms - 50_000,
|
||||
HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 },
|
||||
)
|
||||
.await
|
||||
.is_activated());
|
||||
let before = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
|
||||
.await
|
||||
.expect("valid late lease");
|
||||
let fingerprint = media_uri.rsplit('/').nth(1).expect("asset fingerprint");
|
||||
for (uri, range, expected) in [
|
||||
(media_uri.replace("/0.ts", "/12.ts"), None, StatusCode::NOT_FOUND),
|
||||
(media_uri.replace(fingerprint, "wrong-fingerprint"), None, StatusCode::NOT_FOUND),
|
||||
(media_uri.replace(&fixture.access_lease_id.0, "unknown-lease"), None, StatusCode::NOT_FOUND),
|
||||
(media_uri.clone(), Some("bytes=1000000000000-"), StatusCode::RANGE_NOT_SATISFIABLE),
|
||||
] {
|
||||
let response = custom_media_response(Arc::clone(&fixture.app_state), &uri, range).await;
|
||||
assert_eq!(response.status(), expected);
|
||||
}
|
||||
fixture.app_state.app_config.api_proxy.store(None);
|
||||
let response = custom_media_response(Arc::clone(&fixture.app_state), &media_uri, None).await;
|
||||
assert_eq!(response.status(), fixture.app_state.app_config.get_auth_error_status());
|
||||
let after = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.access_lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
super::super::current_time_millis(),
|
||||
)
|
||||
.await
|
||||
.expect("unchanged late lease");
|
||||
assert_eq!(after.last_seen_at_ms, before.last_seen_at_ms);
|
||||
assert_eq!(after.valid_until_ms, before.valid_until_ms);
|
||||
assert_eq!(after.active_until_ms, before.active_until_ms);
|
||||
}
|
||||
|
||||
async fn committed_late_http_fixture() -> (terminal_tail::PublicationLateFixture, String) {
|
||||
let fixture = terminal_tail::publication_late_fixture().await;
|
||||
{
|
||||
let mut session = fixture.session.write().await;
|
||||
session.segments.clear();
|
||||
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
|
||||
}
|
||||
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest");
|
||||
let media_uri = body.lines().find(|line| line.contains("/cvs/hls/")).expect("custom segment URI").to_owned();
|
||||
(fixture, media_uri)
|
||||
}
|
||||
|
||||
async fn custom_media_response(app_state: Arc<AppState>, uri: &str, range: Option<&str>) -> Response<Body> {
|
||||
custom_media_response_with_method(app_state, uri, range, Method::GET).await
|
||||
}
|
||||
|
||||
async fn custom_media_response_with_method(
|
||||
app_state: Arc<AppState>,
|
||||
uri: &str,
|
||||
range: Option<&str>,
|
||||
method: Method,
|
||||
) -> Response<Body> {
|
||||
let router = crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(app_state);
|
||||
let mut request = Request::builder().method(method).uri(uri);
|
||||
if let Some(range) = range {
|
||||
request = request.header(header::RANGE, range);
|
||||
}
|
||||
router
|
||||
.oneshot(request.extension(ConnectInfo(test_addr())).body(Body::empty()).expect("custom media request"))
|
||||
.await
|
||||
.expect("custom media response")
|
||||
}
|
||||
@@ -17,11 +17,14 @@ use std::sync::{
|
||||
const ORIGIN_WINDOW_SEGMENTS: u64 = 6;
|
||||
const SEGMENT_BODY: &[u8] = &[0x47; 8_192];
|
||||
|
||||
fn origin_playlist(head: u64) -> Vec<u8> {
|
||||
fn origin_playlist(head: u64) -> Vec<u8> { origin_playlist_with_duration(head, 2) }
|
||||
|
||||
pub(super) fn origin_playlist_with_duration(head: u64, target_duration: u64) -> Vec<u8> {
|
||||
use std::fmt::Write;
|
||||
let mut body = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXT-X-MEDIA-SEQUENCE:{head}\n");
|
||||
let mut body =
|
||||
format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{target_duration}\n#EXT-X-MEDIA-SEQUENCE:{head}\n");
|
||||
for sequence in head..head + ORIGIN_WINDOW_SEGMENTS {
|
||||
let _ = write!(body, "#EXTINF:2.0,\nseg-{sequence}.ts\n");
|
||||
let _ = write!(body, "#EXTINF:{target_duration}.0,\nseg-{sequence}.ts\n");
|
||||
}
|
||||
body.into_bytes()
|
||||
}
|
||||
@@ -61,6 +64,20 @@ async fn first_origin_sequence_for_new_lease(
|
||||
request_url: &str,
|
||||
lease: &str,
|
||||
) -> u64 {
|
||||
let body = open_manifest_for_new_lease(app_state, input, request_url, lease).await;
|
||||
let session_key = super::super::build_hls_origin_source(input, "12345").session_key();
|
||||
let first_proxy_seq = manifest_media_sequence(&body);
|
||||
let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session");
|
||||
let session = session.read().await;
|
||||
session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence
|
||||
}
|
||||
|
||||
pub(super) async fn open_manifest_for_new_lease(
|
||||
app_state: &Arc<AppState>,
|
||||
input: &ConfigInput,
|
||||
request_url: &str,
|
||||
lease: &str,
|
||||
) -> String {
|
||||
let origin_source = super::super::build_hls_origin_source(input, "12345");
|
||||
let session_key = origin_source.session_key();
|
||||
let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret());
|
||||
@@ -88,11 +105,7 @@ async fn first_origin_sequence_for_new_lease(
|
||||
.await
|
||||
.expect("hls cache should handle the live entrypoint");
|
||||
assert_eq!(response.status(), StatusCode::OK, "lease {lease}");
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8");
|
||||
let first_proxy_seq = manifest_media_sequence(&body);
|
||||
let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session");
|
||||
let session = session.read().await;
|
||||
session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence
|
||||
String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8")
|
||||
}
|
||||
|
||||
/// Moves the retained session's last origin confirmation `idle_ms` into the
|
||||
|
||||
@@ -0,0 +1,526 @@
|
||||
use super::{support::TestEncodedManifestOrigin, *};
|
||||
use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode};
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use tuliprox_hls::api::HlsFreshManifestRequiredReason;
|
||||
|
||||
struct ShortWindowFixture {
|
||||
_temp_dir: tempfile::TempDir,
|
||||
origin: TestSegmentOrigin,
|
||||
app_state: Arc<AppState>,
|
||||
input: ConfigInput,
|
||||
request_url: String,
|
||||
head: Arc<AtomicU64>,
|
||||
count: Arc<AtomicUsize>,
|
||||
proxy_session_id: ProxySessionId,
|
||||
}
|
||||
|
||||
enum ShortWindowResources {
|
||||
Plain,
|
||||
EncryptedWithMap,
|
||||
FirstReady { media_base_url: String },
|
||||
}
|
||||
|
||||
async fn short_window_fixture(count: usize) -> ShortWindowFixture {
|
||||
short_window_resource_fixture(count, ShortWindowResources::Plain).await
|
||||
}
|
||||
|
||||
async fn short_window_resource_fixture(count: usize, resources: ShortWindowResources) -> ShortWindowFixture {
|
||||
let startup_mode = match &resources {
|
||||
ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => HlsStartupMode::Conservative,
|
||||
ShortWindowResources::FirstReady { .. } => HlsStartupMode::FirstReady,
|
||||
};
|
||||
let head = Arc::new(AtomicU64::new(100));
|
||||
let count = Arc::new(AtomicUsize::new(count));
|
||||
let (origin_head, origin_count) = (Arc::clone(&head), Arc::clone(&count));
|
||||
let origin = spawn_test_binary_origin(Arc::new(move |path| {
|
||||
let body = if path_has_extension(path, "m3u8") {
|
||||
let head = origin_head.load(Ordering::SeqCst);
|
||||
let mut body =
|
||||
format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n");
|
||||
let extension = match &resources {
|
||||
ShortWindowResources::Plain | ShortWindowResources::FirstReady { .. } => "ts",
|
||||
ShortWindowResources::EncryptedWithMap => {
|
||||
body.push_str("#EXT-X-MAP:URI=\"init.mp4\"\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",IV=0x01010101010101010101010101010101\n");
|
||||
"m4s"
|
||||
}
|
||||
};
|
||||
for offset in 0..origin_count.load(Ordering::SeqCst) {
|
||||
let segment_url = match &resources {
|
||||
ShortWindowResources::FirstReady { media_base_url } => {
|
||||
format!("{media_base_url}/segment-{}.ts", head + u64::try_from(offset).unwrap())
|
||||
}
|
||||
ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => {
|
||||
format!("segment-{}.{extension}", head + u64::try_from(offset).unwrap())
|
||||
}
|
||||
};
|
||||
writeln!(body, "#EXTINF:2.5,\n{segment_url}").unwrap();
|
||||
}
|
||||
Arc::from(body.into_bytes())
|
||||
} else if path_has_extension(path, "mp4") {
|
||||
Arc::from(&b"short-window-map"[..])
|
||||
} else if path_has_extension(path, "bin") {
|
||||
Arc::from(AES_TEST_KEY_BYTES)
|
||||
} else {
|
||||
match &resources {
|
||||
ShortWindowResources::Plain => Arc::from(&b"short-window-media"[..]),
|
||||
ShortWindowResources::FirstReady { .. } => unreachable!("media uses the gated segment origin"),
|
||||
ShortWindowResources::EncryptedWithMap => Arc::from(encrypt_test_aes128_cbc_pkcs7(
|
||||
b"short-window-media", AES_TEST_KEY_BYTES, [1; 16])),
|
||||
}
|
||||
};
|
||||
TestBinaryOriginResponse::new(StatusCode::OK, body)
|
||||
}))
|
||||
.await;
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let input = ConfigInput {
|
||||
id: 1,
|
||||
name: Arc::from("short-window-input"),
|
||||
input_type: InputType::M3u,
|
||||
url: origin.base_url.clone(),
|
||||
max_connections: 1,
|
||||
enabled: true,
|
||||
..ConfigInput::default()
|
||||
};
|
||||
let app_state = test_app_state_with_hls_proxy_and_inputs(
|
||||
Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)),
|
||||
vec![Arc::new(input.clone())],
|
||||
);
|
||||
let mut cache = HlsCacheConfigDto {
|
||||
cache_path: Some(temp_dir.path().to_string_lossy().to_string()),
|
||||
startup: HlsStartupConfigDto { mode: startup_mode, ..HlsStartupConfigDto::default() },
|
||||
..HlsCacheConfigDto::default()
|
||||
};
|
||||
cache.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Friendly;
|
||||
let config = Config {
|
||||
reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto {
|
||||
hls_cache: Some(cache),
|
||||
..Default::default()
|
||||
})),
|
||||
..Config::default()
|
||||
};
|
||||
app_state.app_config.config.store(Arc::new(config));
|
||||
app_state.hls.proxy.update_config(&app_state.app_config).await;
|
||||
configure_default_test_server(&app_state);
|
||||
let mut target = test_m3u_hls_share_target();
|
||||
target.name = "default".to_string();
|
||||
store_test_sources_with_target(&app_state, input.clone(), target.clone());
|
||||
create_active_hls_user_session(&app_state).await;
|
||||
let request_url = format!("{}/channel/index.m3u8", origin.base_url);
|
||||
cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "short-channel", &request_url)).await;
|
||||
let key = super::super::build_hls_origin_source(&input, "12345").session_key();
|
||||
let proxy_session_id = crate::api::model::build_proxy_session_id(&key, &app_state.get_encrypt_secret());
|
||||
ShortWindowFixture { _temp_dir: temp_dir, origin, app_state, input, request_url, head, count, proxy_session_id }
|
||||
}
|
||||
|
||||
/// Keeps the first media fetch pending until all lease entries have been checked.
|
||||
async fn gated_segment_origin(gate: Arc<tokio::sync::Semaphore>) -> TestEncodedManifestOrigin {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let requests = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let origin_requests = Arc::clone(&requests);
|
||||
let task = tokio::spawn(async move {
|
||||
while let Ok((mut socket, _)) = listener.accept().await {
|
||||
let requests = Arc::clone(&origin_requests);
|
||||
let gate = Arc::clone(&gate);
|
||||
tokio::spawn(async move {
|
||||
let mut buffer = [0_u8; 2048];
|
||||
let read = socket.read(&mut buffer).await.unwrap();
|
||||
requests.lock().await.push(String::from_utf8_lossy(&buffer[..read]).to_string());
|
||||
let _permit = gate.acquire().await;
|
||||
let body = [0x47; 8192];
|
||||
let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
|
||||
socket.write_all(headers.as_bytes()).await.unwrap();
|
||||
socket.write_all(&body).await.unwrap();
|
||||
});
|
||||
}
|
||||
});
|
||||
TestEncodedManifestOrigin { base_url: format!("http://{addr}"), requests, task }
|
||||
}
|
||||
|
||||
async fn wait_for_shared_first_ready_refresh(
|
||||
fixture: &ShortWindowFixture,
|
||||
session: &HlsSessionHandle,
|
||||
previous_requests: usize,
|
||||
) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if fixture.origin.manifest_request_count() > previous_requests
|
||||
&& !session.read().await.origin_refresh.in_flight
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("additional leases complete one shared freshness request");
|
||||
assert_eq!(fixture.origin.manifest_request_count(), previous_requests + 1);
|
||||
}
|
||||
|
||||
async fn assert_first_ready_lease_media(
|
||||
fixture: &ShortWindowFixture,
|
||||
leases: &mut tokio::task::JoinSet<String>,
|
||||
minimum_media_sequence: u64,
|
||||
) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while let Some(result) = leases.join_next().await {
|
||||
let body = result.unwrap();
|
||||
assert!(manifest_media_sequence(&body) >= minimum_media_sequence);
|
||||
assert_eq!(media_uri_count(&body), 1);
|
||||
for uri in body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()) {
|
||||
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response_body(response).await.as_ref(), &[0x47; 8192]);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("media publication releases every waiting lease");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_ready_leases_share_qualification_while_first_segment_is_loading() {
|
||||
let gate = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let media = gated_segment_origin(Arc::clone(&gate)).await;
|
||||
let fixture = Arc::new(
|
||||
short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() })
|
||||
.await,
|
||||
);
|
||||
let mut leases = tokio::task::JoinSet::new();
|
||||
let first = Arc::clone(&fixture);
|
||||
leases.spawn(async move { open_short_lease(&first, "first-ready-0").await });
|
||||
let session = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if !media.requests.lock().await.is_empty() {
|
||||
if let Some(session) =
|
||||
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await
|
||||
{
|
||||
if !session.read().await.origin_refresh.in_flight {
|
||||
break session;
|
||||
}
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("qualified origin has started its first segment fetch");
|
||||
let generation = {
|
||||
let session = session.read().await;
|
||||
assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady);
|
||||
assert!(session.last_rendered_manifest.is_none());
|
||||
assert_eq!(session.origin_control.acceptance_generation.0, 1);
|
||||
session.origin_control.acceptance_generation
|
||||
};
|
||||
assert_eq!(fixture.origin.manifest_request_count(), 3);
|
||||
for lease in 1..20 {
|
||||
let fixture = Arc::clone(&fixture);
|
||||
leases.spawn(async move { open_short_lease(&fixture, &format!("first-ready-{lease}")).await });
|
||||
}
|
||||
wait_for_shared_first_ready_refresh(&fixture, &session, 3).await;
|
||||
assert!(session.read().await.last_rendered_manifest.is_none());
|
||||
assert_eq!(session.read().await.origin_control.acceptance_generation, generation);
|
||||
gate.close();
|
||||
assert_first_ready_lease_media(&fixture, &mut leases, 0).await;
|
||||
assert_eq!(session.read().await.origin_control.acceptance_generation, generation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_ready_revalidation_keeps_its_qualification_while_new_media_is_loading() {
|
||||
let gate = Arc::new(tokio::sync::Semaphore::new(1));
|
||||
let media = gated_segment_origin(Arc::clone(&gate)).await;
|
||||
let fixture = Arc::new(
|
||||
short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() })
|
||||
.await,
|
||||
);
|
||||
let previous_body = open_short_lease(&fixture, "revalidation-baseline").await;
|
||||
let session =
|
||||
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap();
|
||||
let previous_commit = {
|
||||
let mut session = session.write().await;
|
||||
assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady);
|
||||
assert_eq!(session.origin_control.acceptance_generation.0, 1);
|
||||
session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0;
|
||||
session.activity.last_authorized_media_at_ms = None;
|
||||
session.startup.as_mut().unwrap().revisions.clear();
|
||||
for segment in session.segments.values_mut() {
|
||||
segment.status = SegmentCacheStatus::Expired;
|
||||
}
|
||||
session.last_normal_manifest_commit_identity().unwrap()
|
||||
};
|
||||
assert_eq!(fixture.origin.manifest_request_count(), 3);
|
||||
assert_eq!(gate.forget_permits(1), 1);
|
||||
assert_eq!(
|
||||
hls_manifest_commit_requirement(
|
||||
&session,
|
||||
tuliprox_hls::api::HlsSessionStoreOutcome::Reused,
|
||||
None,
|
||||
true,
|
||||
super::super::current_time_millis(),
|
||||
)
|
||||
.await,
|
||||
HlsManifestCommitRequirement::FreshCommitRequired {
|
||||
reason: HlsFreshManifestRequiredReason::ExpiredRevalidation,
|
||||
}
|
||||
);
|
||||
fixture.head.store(101, Ordering::SeqCst);
|
||||
let mut leases = tokio::task::JoinSet::new();
|
||||
let first = Arc::clone(&fixture);
|
||||
leases.spawn(async move { open_short_lease(&first, "revalidation-waiting-0").await });
|
||||
let revalidation = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if media.requests.lock().await.len() >= 2 {
|
||||
let state = session.read().await;
|
||||
if !state.origin_refresh.in_flight && state.origin_seq_highwater == Some(101) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let media_requests = media.requests.lock().await.len();
|
||||
{
|
||||
let session = session.read().await;
|
||||
assert!(
|
||||
revalidation.is_ok(),
|
||||
"revalidation requests={} generation={} highwater={:?} media_requests={media_requests} in_flight={} commit={:?}",
|
||||
fixture.origin.manifest_request_count(),
|
||||
session.origin_control.acceptance_generation.0,
|
||||
session.origin_seq_highwater,
|
||||
session.origin_refresh.in_flight,
|
||||
session.last_normal_manifest_commit_identity(),
|
||||
);
|
||||
}
|
||||
assert_eq!(fixture.origin.manifest_request_count(), 5);
|
||||
{
|
||||
let session = session.read().await;
|
||||
assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady);
|
||||
assert_eq!(session.origin_control.acceptance_generation.0, 2);
|
||||
assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit));
|
||||
}
|
||||
assert_eq!(
|
||||
hls_manifest_commit_requirement(
|
||||
&session,
|
||||
tuliprox_hls::api::HlsSessionStoreOutcome::Reused,
|
||||
None,
|
||||
true,
|
||||
super::super::current_time_millis(),
|
||||
)
|
||||
.await,
|
||||
HlsManifestCommitRequirement::FreshCommitRequired {
|
||||
reason: HlsFreshManifestRequiredReason::ExpiredRevalidation,
|
||||
}
|
||||
);
|
||||
for lease in 1..20 {
|
||||
let fixture = Arc::clone(&fixture);
|
||||
leases.spawn(async move { open_short_lease(&fixture, &format!("revalidation-waiting-{lease}")).await });
|
||||
}
|
||||
wait_for_shared_first_ready_refresh(&fixture, &session, 5).await;
|
||||
assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2);
|
||||
assert_eq!(session.read().await.last_normal_manifest_commit_identity(), Some(previous_commit));
|
||||
gate.close();
|
||||
assert_first_ready_lease_media(&fixture, &mut leases, manifest_media_sequence(&previous_body) + 1).await;
|
||||
assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2);
|
||||
assert!(session.read().await.last_normal_manifest_commit_identity().unwrap() > previous_commit);
|
||||
}
|
||||
|
||||
async fn open_short_lease(fixture: &ShortWindowFixture, lease: &str) -> String {
|
||||
let access_lease_id = HlsAccessLeaseId(lease.to_string());
|
||||
let access = test_hls_access_context(fixture.proxy_session_id.clone(), access_lease_id.clone());
|
||||
prepare_pending_test_hls_access_lease(&fixture.app_state, &fixture.proxy_session_id, &access_lease_id).await;
|
||||
let response = super::super::try_hls_cache_canonical_manifest_response(
|
||||
&fixture.app_state,
|
||||
&test_fingerprint(),
|
||||
&access,
|
||||
&fixture.proxy_session_id,
|
||||
&access_lease_id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
super::super::HlsCacheManifestOrigin {
|
||||
raw_request_url: &fixture.request_url,
|
||||
session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&fixture.request_url),
|
||||
input: &fixture.input,
|
||||
origin_source: super::super::build_hls_origin_source(&fixture.input, "12345"),
|
||||
},
|
||||
HeaderMap::new(),
|
||||
None,
|
||||
"/m3u-stream/live/hls-user/hls-pass/12345.m3u8",
|
||||
super::super::HlsManifestRefreshOrdering::Background,
|
||||
)
|
||||
.await
|
||||
.expect("shared HLS response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(!response.headers().contains_key(header::RETRY_AFTER));
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
assert!(!body.contains("#EXT-X-ENDLIST"), "short origin must remain live: {body}");
|
||||
assert!(body.contains("/hls/shared/live/"));
|
||||
body
|
||||
}
|
||||
|
||||
async fn assert_short_media(fixture: &ShortWindowFixture, body: &str) {
|
||||
for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) {
|
||||
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response_body(response).await.as_ref(), b"short-window-media");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn one_segment_origin_publishes_live_manifest_and_segment() {
|
||||
let fixture = short_window_fixture(1).await;
|
||||
let body = open_short_lease(&fixture, "short-one").await;
|
||||
assert_eq!(media_uri_count(&body), 1);
|
||||
assert_short_media(&fixture, &body).await;
|
||||
assert_eq!(fixture.origin.manifest_request_count(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_segment_origin_publishes_live_manifest_and_segments() {
|
||||
let fixture = short_window_fixture(2).await;
|
||||
let body = open_short_lease(&fixture, "short-two").await;
|
||||
assert_eq!(media_uri_count(&body), 2);
|
||||
assert_short_media(&fixture, &body).await;
|
||||
assert_eq!(fixture.origin.manifest_request_count(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn twenty_actual_leases_share_the_same_short_origin_qualification() {
|
||||
let mut counts = Vec::new();
|
||||
for leases in [1, 20] {
|
||||
let fixture = Arc::new(short_window_fixture(1).await);
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
for lease in 0..leases {
|
||||
let fixture = Arc::clone(&fixture);
|
||||
tasks.spawn(async move { open_short_lease(&fixture, &format!("short-shared-{lease}")).await });
|
||||
}
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
let body = result.unwrap();
|
||||
assert_eq!(media_uri_count(&body), 1);
|
||||
assert_short_media(&fixture, &body).await;
|
||||
}
|
||||
counts.push(fixture.origin.manifest_request_count());
|
||||
let before_new_lease = fixture.origin.manifest_request_count();
|
||||
open_short_lease(&fixture, "later-short-lease").await;
|
||||
assert_eq!(fixture.origin.manifest_request_count(), before_new_lease);
|
||||
}
|
||||
assert_eq!(counts, vec![3, 3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ten_short_refresh_cycles_keep_serving_live_media_and_immutable_old_routes() {
|
||||
assert_ten_short_refresh_cycles([3, 1, 3, 1, 3, 1, 3, 1, 3, 1]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuously_advancing_one_segment_origin_serves_ten_refresh_cycles() {
|
||||
assert_ten_short_refresh_cycles([1; 10]).await;
|
||||
}
|
||||
|
||||
async fn assert_ten_short_refresh_cycles(counts: [usize; 10]) {
|
||||
let fixture = short_window_fixture(1).await;
|
||||
let initial = open_short_lease(&fixture, "short-reloads").await;
|
||||
assert_short_media(&fixture, &initial).await;
|
||||
let uri = format!("/hls/shared/live/{}/short-reloads/manifest.m3u8", fixture.proxy_session_id.0);
|
||||
let session =
|
||||
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap();
|
||||
let mut previous = initial.clone();
|
||||
let mut previous_last = session.read().await.last_rendered_manifest.as_ref().unwrap().last_proxy_seq;
|
||||
for cycle in 1..=10 {
|
||||
fixture.head.store(100 + cycle * 4, Ordering::SeqCst);
|
||||
fixture.count.store(counts[usize::try_from(cycle - 1).unwrap()], Ordering::SeqCst);
|
||||
{
|
||||
let mut session = session.write().await;
|
||||
session.origin_refresh.next_fetch_allowed_at_ms = 0;
|
||||
}
|
||||
let mut current = None;
|
||||
for _ in 0..100 {
|
||||
let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
assert!(!body.contains("#EXT-X-ENDLIST"));
|
||||
if body != previous {
|
||||
current = Some(body);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
let body = current.expect("normal refresh advances the short window");
|
||||
assert!(manifest_media_sequence(&body) >= manifest_media_sequence(&previous));
|
||||
previous = body.clone();
|
||||
assert_short_media(&fixture, &body).await;
|
||||
let lease = fixture
|
||||
.app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(
|
||||
&HlsAccessLeaseId("short-reloads".to_string()),
|
||||
&fixture.proxy_session_id,
|
||||
super::super::current_time_millis(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let snapshot = lease.last_manifest_snapshot.unwrap();
|
||||
assert!(snapshot.last_proxy_seq > previous_last);
|
||||
previous_last = snapshot.last_proxy_seq;
|
||||
assert_eq!(snapshot.playlist_duration_ms, u64::try_from(media_uri_count(&body)).unwrap() * 2_500);
|
||||
}
|
||||
assert_eq!(fixture.origin.manifest_request_count(), 13);
|
||||
assert_short_media(&fixture, &initial).await;
|
||||
let invalid = uri.replace("short-reloads", "missing-lease");
|
||||
assert_ne!(get_response(Arc::clone(&fixture.app_state), &invalid, None).await.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_manifests_keep_map_key_and_segment_routes_on_later_publications() {
|
||||
let fixture = short_window_resource_fixture(2, ShortWindowResources::EncryptedWithMap).await;
|
||||
let initial = open_short_lease(&fixture, "short-resources").await;
|
||||
let session =
|
||||
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap();
|
||||
for uri in initial.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) {
|
||||
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
response_body(response).await;
|
||||
}
|
||||
let mut bodies = vec![initial];
|
||||
fixture.head.store(102, Ordering::SeqCst);
|
||||
session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0;
|
||||
let uri = format!("/hls/shared/live/{}/short-resources/manifest.m3u8", fixture.proxy_session_id.0);
|
||||
for _ in 0..100 {
|
||||
let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
if body != bodies[0] {
|
||||
bodies.push(body);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
assert_eq!(bodies.len(), 2, "a second short publication must advance");
|
||||
for body in bodies {
|
||||
assert!(!body.contains("#EXT-X-ENDLIST"));
|
||||
for (tag, expected) in [("#EXT-X-MAP:", &b"short-window-map"[..]), ("#EXT-X-KEY:", AES_TEST_KEY_BYTES)] {
|
||||
let uri = body
|
||||
.lines()
|
||||
.find(|line| line.starts_with(tag))
|
||||
.unwrap()
|
||||
.split_once("URI=\"")
|
||||
.unwrap()
|
||||
.1
|
||||
.split_once('"')
|
||||
.unwrap()
|
||||
.0;
|
||||
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response_body(response).await.as_ref(), expected);
|
||||
}
|
||||
for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) {
|
||||
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response_body(response).await.as_ref(),
|
||||
encrypt_test_aes128_cbc_pkcs7(b"short-window-media", AES_TEST_KEY_BYTES, [1; 16])
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(fixture.origin.manifest_request_count(), 4);
|
||||
}
|
||||
@@ -420,7 +420,7 @@ async fn hls_cache_manifest_cold_start_synchronously_returns_initial_manifest()
|
||||
assert_eq!(binding.input_name.as_ref(), "test-input");
|
||||
assert_eq!(binding.account_name.as_ref(), "test-input");
|
||||
}
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -491,7 +491,7 @@ async fn tokens_without_entry_kind_keep_bad_request_without_session() {
|
||||
let response = fixture.token_request(&client("10.0.0.1", 50_121), &token, HeaderMap::new()).await;
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
assert!(fixture.origin.requests().is_empty());
|
||||
assert_eq!(fixture.origin.requests().len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -507,7 +507,7 @@ async fn shared_hls_cache_on_never_recreates_from_token() {
|
||||
);
|
||||
let response = fixture.token_request(&client("10.0.0.1", 50_131), &token, HeaderMap::new()).await;
|
||||
assert_ne!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert!(fixture.origin.requests().is_empty());
|
||||
assert_eq!(fixture.origin.requests().len(), 0);
|
||||
assert!(fixture.app_state.active_users.get_and_update_user_session("hls-user", hint).await.is_none());
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,9 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_hls_origin_io_context(
|
||||
reservation_ttl_secs: 60,
|
||||
preacquired_provider_handle: None,
|
||||
started_generation: None,
|
||||
prepared_account: None,
|
||||
playback_kind: tuliprox_core::model::PlaybackKind::LiveHls,
|
||||
account_runtime: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -156,8 +156,8 @@ fn hls_soft_overlap_delay_scales_with_tuliprox_target_pressure() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_account_overlap_reclaim_preempts_speculative_session() {
|
||||
let app_state = test_app_state();
|
||||
let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() };
|
||||
let input = overlap_provider_input();
|
||||
let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
|
||||
let winner = create_bound_hls_test_session(&app_state, &input, "winner", "account-a", 1_000).await;
|
||||
let loser = create_bound_hls_test_session(&app_state, &input, "loser", "account-a", 1_000).await;
|
||||
let winner_proxy_session_id = winner.read().await.proxy_session_id.clone();
|
||||
@@ -175,7 +175,9 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() {
|
||||
}
|
||||
let loser_generation = loser.read().await.activity.origin_work_generation;
|
||||
|
||||
super::super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await;
|
||||
winner.write().await.mark_authorized_media_access(10_000);
|
||||
winner.write().await.activity.last_delivered_media_at_ms = Some(10_000);
|
||||
drop(prepare_overlap_origin(&app_state, &input, &winner, 10_000).await);
|
||||
|
||||
assert!(app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000).await);
|
||||
assert!(!app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000).await);
|
||||
@@ -191,8 +193,8 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_account_overlap_promotes_speculative_session_after_soft_window() {
|
||||
let app_state = test_app_state();
|
||||
let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() };
|
||||
let input = overlap_provider_input();
|
||||
let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
|
||||
let displaced = create_bound_hls_test_session(&app_state, &input, "displaced", "account-a", 1_000).await;
|
||||
let promoted = create_bound_hls_test_session(&app_state, &input, "promoted", "account-a", 1_000).await;
|
||||
let displaced_proxy_session_id = displaced.read().await.proxy_session_id.clone();
|
||||
@@ -210,7 +212,9 @@ async fn hls_account_overlap_promotes_speculative_session_after_soft_window() {
|
||||
}
|
||||
let displaced_generation = displaced.read().await.activity.origin_work_generation;
|
||||
|
||||
super::super::promote_elapsed_hls_account_overlaps(&app_state, 20_001).await;
|
||||
promoted.write().await.mark_authorized_media_access(20_001);
|
||||
promoted.write().await.activity.last_delivered_media_at_ms = Some(20_001);
|
||||
drop(prepare_overlap_origin(&app_state, &input, &promoted, 20_001).await);
|
||||
|
||||
let displaced_binding_mode = displaced.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone();
|
||||
assert!(matches!(
|
||||
@@ -262,8 +266,7 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work(
|
||||
.await
|
||||
.expect("interactive work should use soft-active overlap before grace");
|
||||
|
||||
let binding =
|
||||
prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared");
|
||||
let binding = new_session.read().await.origin_account_binding.clone().expect("speculative binding is published");
|
||||
assert_eq!(binding.account_name, input.name);
|
||||
assert!(matches!(
|
||||
&binding.binding_mode,
|
||||
@@ -273,11 +276,15 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work(
|
||||
} if displaced_proxy_session_id == &old_proxy_session_id
|
||||
));
|
||||
assert!(matches!(
|
||||
prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation),
|
||||
prepared_origin
|
||||
.preacquired_origin_account_handle
|
||||
.as_ref()
|
||||
.and_then(tuliprox_session::ManagedProviderHandle::handle)
|
||||
.map(|handle| &handle.allocation),
|
||||
Some(super::super::ProviderAllocation::Available(_))
|
||||
));
|
||||
|
||||
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -324,14 +331,16 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() {
|
||||
.await
|
||||
.expect("normal HLS policy should preempt active soft HLS binding");
|
||||
|
||||
let new_binding = prepared_origin
|
||||
.origin_account_binding_to_store
|
||||
.as_ref()
|
||||
.expect("preempting session should receive active binding");
|
||||
let new_binding =
|
||||
normal_session.read().await.origin_account_binding.clone().expect("preempting session receives active binding");
|
||||
assert_eq!(new_binding.account_name, input.name);
|
||||
assert!(matches!(new_binding.binding_mode, HlsOriginAccountBindingMode::Active));
|
||||
assert!(matches!(
|
||||
prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation),
|
||||
prepared_origin
|
||||
.preacquired_origin_account_handle
|
||||
.as_ref()
|
||||
.and_then(tuliprox_session::ManagedProviderHandle::handle)
|
||||
.map(|handle| &handle.allocation),
|
||||
Some(super::super::ProviderAllocation::Available(_))
|
||||
));
|
||||
let soft_session = soft_session.read().await;
|
||||
@@ -345,7 +354,7 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() {
|
||||
));
|
||||
drop(soft_session);
|
||||
|
||||
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -437,7 +446,6 @@ async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal
|
||||
|
||||
let result = super::super::prepare_hls_origin_policy_preempt_runtime(
|
||||
&app_state,
|
||||
&soft_session,
|
||||
&input,
|
||||
"http://account.example.com/live/account-user/account-pass/soft.m3u8",
|
||||
"http://account.example.com/live/account-user/account-pass/soft.m3u8",
|
||||
@@ -496,12 +504,18 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai
|
||||
.expect("interactive work can use grace when normal acquire and overlap fail");
|
||||
|
||||
assert!(matches!(
|
||||
prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation),
|
||||
prepared_origin
|
||||
.preacquired_origin_account_handle
|
||||
.as_ref()
|
||||
.and_then(tuliprox_session::ManagedProviderHandle::handle)
|
||||
.map(|handle| &handle.allocation),
|
||||
Some(super::super::ProviderAllocation::GracePeriod(_))
|
||||
));
|
||||
assert_eq!(
|
||||
prepared_origin
|
||||
.origin_account_binding_to_store
|
||||
session
|
||||
.read()
|
||||
.await
|
||||
.origin_account_binding
|
||||
.as_ref()
|
||||
.expect("grace binding should still bind the selected account")
|
||||
.account_name
|
||||
@@ -509,7 +523,7 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai
|
||||
input.name.as_ref()
|
||||
);
|
||||
|
||||
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
app_state.connection_manager.release_provider_handle(Some(occupied));
|
||||
}
|
||||
|
||||
@@ -715,3 +729,40 @@ fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_
|
||||
super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted
|
||||
);
|
||||
}
|
||||
|
||||
async fn prepare_overlap_origin(
|
||||
app: &Arc<crate::api::model::AppState>,
|
||||
input: &ConfigInput,
|
||||
session: &tuliprox_hls::api::HlsSessionHandle,
|
||||
now_ms: u64,
|
||||
) -> super::super::session::PreparedHlsOriginRuntime {
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
super::super::prepare_hls_origin_runtime(
|
||||
app,
|
||||
session,
|
||||
input,
|
||||
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
|
||||
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
|
||||
&id,
|
||||
&test_fingerprint(),
|
||||
ConnectionKind::Normal,
|
||||
0,
|
||||
super::super::HlsOriginWorkKind::Manifest,
|
||||
super::super::HlsOriginWorkClass::ManifestInteractive,
|
||||
now_ms,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn healthy_prepare_detaches_an_unprotected_expired_binding_before_reacquisition() {
|
||||
let input = overlap_provider_input();
|
||||
let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
|
||||
let session = create_bound_hls_test_session(&app, &input, "expired-protection", "account-a", 1_000).await;
|
||||
let generation = session.read().await.activity.origin_work_generation;
|
||||
drop(prepare_overlap_origin(&app, &input, &session, 10_000).await);
|
||||
let current = session.read().await;
|
||||
assert!(current.activity.origin_work_generation > generation);
|
||||
assert!(current.origin_account_binding.as_ref().unwrap().generation > 0);
|
||||
}
|
||||
|
||||
@@ -21,6 +21,33 @@ use http_body_util::BodyExt;
|
||||
use shared::model::{HlsStripMode, InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
#[tokio::test]
|
||||
async fn origin_resource_no_account_available_includes_retry_after() {
|
||||
let app_state = test_app_state();
|
||||
let access_context = test_hls_access_context(
|
||||
ProxySessionId("shared-hls-session".to_string()),
|
||||
HlsAccessLeaseId("access-lease".to_string()),
|
||||
);
|
||||
|
||||
for reason in [
|
||||
super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted,
|
||||
super::super::HlsOriginRuntimeNoAccountReason::OriginBindingPreempted,
|
||||
] {
|
||||
let response = super::super::hls_origin_runtime_resource_failure_response(
|
||||
&app_state,
|
||||
&access_context,
|
||||
super::super::HlsOriginRuntimeAcquireError::NoAccountAvailable { reason },
|
||||
);
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{reason:?}");
|
||||
assert_eq!(
|
||||
response.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()),
|
||||
Some("2"),
|
||||
"{reason:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_origin_binding_requires_runtime_prepare_for_missing_or_detached_account() {
|
||||
let input = single_hls_provider_input("known-account");
|
||||
@@ -52,7 +79,7 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho
|
||||
}
|
||||
let old_generation = session.read().await.activity.origin_work_generation;
|
||||
|
||||
super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 4_001).await;
|
||||
super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 6_001).await;
|
||||
|
||||
{
|
||||
let session = session.read().await;
|
||||
@@ -77,17 +104,16 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho
|
||||
0,
|
||||
super::super::HlsOriginWorkKind::Manifest,
|
||||
super::super::HlsOriginWorkClass::ManifestInteractive,
|
||||
4_100,
|
||||
6_100,
|
||||
)
|
||||
.await
|
||||
.expect("authorized origin work can reacquire a detached binding");
|
||||
|
||||
let binding =
|
||||
prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller");
|
||||
let binding = session.read().await.origin_account_binding.clone().expect("new binding is published atomically");
|
||||
assert_eq!(binding.account_name.as_ref(), "account-a");
|
||||
assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active));
|
||||
assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8");
|
||||
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -122,6 +148,70 @@ async fn hls_account_binding_without_media_activity_expires_after_startup_window
|
||||
assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_hls_binding_has_bounded_startup_protection_despite_old_media_delivery() {
|
||||
let input = overlap_provider_input();
|
||||
let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
|
||||
let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await;
|
||||
{
|
||||
let mut s = session.write().await;
|
||||
s.target_duration = Some(11);
|
||||
s.activity.last_authorized_media_at_ms = Some(70_000);
|
||||
s.activity.last_delivered_media_at_ms = Some(70_000);
|
||||
}
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
for now in [200_001, 204_999, 205_000] {
|
||||
let prepared = super::super::prepare_hls_origin_runtime(
|
||||
&app,
|
||||
&session,
|
||||
&input,
|
||||
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
|
||||
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
|
||||
&proxy_id,
|
||||
&test_fingerprint(),
|
||||
ConnectionKind::Normal,
|
||||
0,
|
||||
super::super::HlsOriginWorkKind::Resource,
|
||||
super::super::HlsOriginWorkClass::Demand,
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let s = session.read().await;
|
||||
let binding = s.origin_account_binding.as_ref().unwrap();
|
||||
assert!(binding.is_active());
|
||||
assert_eq!(binding.pinned_at_ms, 200_000, "preparing the same binding cannot extend startup protection");
|
||||
assert_eq!(binding.generation, 0);
|
||||
assert_eq!(s.activity.last_delivered_media_at_ms, Some(70_000));
|
||||
drop(s);
|
||||
app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle);
|
||||
}
|
||||
super::super::detach_unprotected_hls_origin_account_bindings(&app, 205_001).await;
|
||||
assert!(
|
||||
session.read().await.origin_account_binding.as_ref().unwrap().is_detached(),
|
||||
"no media progress: releasable after five seconds"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_binding_expired_startup_protects_running_origin_work() {
|
||||
let input = overlap_provider_input();
|
||||
let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
|
||||
let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await;
|
||||
let generation = {
|
||||
let mut s = session.write().await;
|
||||
s.target_duration = Some(11);
|
||||
s.activity.last_authorized_media_at_ms = Some(70_000);
|
||||
s.activity.last_delivered_media_at_ms = Some(70_000);
|
||||
s.start_origin_work()
|
||||
};
|
||||
super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_000).await;
|
||||
assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_active());
|
||||
assert!(session.write().await.finish_origin_work(generation));
|
||||
super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_001).await;
|
||||
assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() {
|
||||
let app_state = test_app_state();
|
||||
@@ -145,6 +235,7 @@ async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_d
|
||||
host_local_index: 123,
|
||||
},
|
||||
proxy_seq: 123,
|
||||
media_start_ms: 0,
|
||||
duration_ms: 4_000,
|
||||
proxy_file_ext: "ts".to_string(),
|
||||
content_type: "video/mp2t".to_string(),
|
||||
|
||||
+1
-1
@@ -95,7 +95,7 @@ async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues
|
||||
assert!(matches!(session.maps.get(&ProxyMapId(1)).expect("map remains").status, MapCacheStatus::Queued { .. }));
|
||||
}
|
||||
assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner")));
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
use super::{
|
||||
super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions},
|
||||
manifest_entry_freshness::open_manifest_for_new_lease,
|
||||
origin_account_rebind::{
|
||||
enable_isolated_rebind_cache, reduced_retry_origin, resume_origin_media, ReducedRetryOriginControl,
|
||||
},
|
||||
*,
|
||||
};
|
||||
use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode};
|
||||
use tuliprox_core::utils::current_time_millis;
|
||||
use tuliprox_hls::api::{
|
||||
trigger_origin_refresh_sync, HlsManifestCommitRequirement, HlsOriginAccountSnapshot, HlsPreparedOriginAccount,
|
||||
};
|
||||
|
||||
const WAIT: Duration = Duration::from_secs(15);
|
||||
|
||||
async fn ready(app: &Arc<AppState>, session: &HlsSessionHandle, sequences: &[u64]) {
|
||||
tokio::time::timeout(WAIT, async {
|
||||
loop {
|
||||
let s = session.read().await;
|
||||
if sequences.iter().all(|seq| {
|
||||
s.segments.get(seq).is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. }))
|
||||
}) && s.active_segment_fetches == 0
|
||||
&& !s.origin_refresh.in_flight
|
||||
&& s.activity.active_origin_work_count.load(Ordering::Acquire) == 0
|
||||
{
|
||||
break;
|
||||
}
|
||||
drop(s);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("autonomous segment work and singleflight complete");
|
||||
assert_eq!(app.active_provider.get_provider_connections_count(), 0);
|
||||
}
|
||||
|
||||
async fn deliver(app: &Arc<AppState>, session: &HlsSessionHandle, manifest: &str, expected: &[u64]) {
|
||||
assert!(!manifest.contains("/cvs/hls/"));
|
||||
assert!(!manifest.contains("#EXT-X-ENDLIST"));
|
||||
for sequence in expected {
|
||||
let (proxy_seq, url) = {
|
||||
let s = session.read().await;
|
||||
let (seq, entry) =
|
||||
s.segments.iter().find(|(_, entry)| entry.origin_key.host_local_sequence == *sequence).unwrap();
|
||||
(*seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone())
|
||||
};
|
||||
let uri = manifest
|
||||
.lines()
|
||||
.find(|line| line.ends_with(&format!("/{proxy_seq:06}.ts")))
|
||||
.expect("segment URI is published by Tuliprox");
|
||||
let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let bytes = response_body(response).await;
|
||||
let path = reqwest::Url::parse(&url).unwrap().path().to_string();
|
||||
assert_eq!(bytes.as_ref(), resume_origin_media(&path));
|
||||
eprintln!("app delivered resource={path} bytes={}", bytes.len());
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_lines, reason = "Keep the ordered end-to-end recovery scenario and its assertions together.")]
|
||||
async fn repaired_reduced_retry(mode: HlsStartupMode) {
|
||||
let control = Arc::new(ReducedRetryOriginControl::default());
|
||||
let origin = reduced_retry_origin(Arc::clone(&control)).await;
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let mut input = overlap_provider_input();
|
||||
input.name = Arc::from("account-a");
|
||||
input.priority = 0;
|
||||
input.url.clone_from(&origin.base_url);
|
||||
let b = &mut input.aliases.as_mut().unwrap()[0];
|
||||
b.name = Arc::from("account-b");
|
||||
b.priority = 10;
|
||||
b.url.clone_from(&origin.base_url);
|
||||
let app = test_app_state_with_hls_proxy_and_inputs(
|
||||
Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)),
|
||||
vec![Arc::new(input.clone())],
|
||||
);
|
||||
let available_b = HlsOriginAccountBinding::new(
|
||||
Arc::clone(&input.name),
|
||||
Arc::from("account-b"),
|
||||
&ProxySessionId("availability-check".into()),
|
||||
current_time_millis(),
|
||||
);
|
||||
assert_eq!(
|
||||
tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b),
|
||||
tuliprox_hls::api::HlsOriginAccountStatus::Known
|
||||
);
|
||||
enable_isolated_rebind_cache(&app, cache.path());
|
||||
let mut config = app.app_config.config.load_full().as_ref().clone();
|
||||
let hls = config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap();
|
||||
hls.startup = (&HlsStartupConfigDto { mode, ..Default::default() }).into();
|
||||
hls.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast;
|
||||
app.app_config.config.store(Arc::new(config));
|
||||
enable_channel_unavailable_custom_response(&app);
|
||||
app.hls.proxy.update_config(&app.app_config).await;
|
||||
create_active_hls_user_session(&app).await;
|
||||
let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url);
|
||||
let lease = "reduced-retry-baseline";
|
||||
let _initial = open_manifest_for_new_lease(&app, &input, &url, lease).await;
|
||||
let key = super::super::build_hls_origin_source(&input, "12345").session_key();
|
||||
let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap();
|
||||
ready(&app, &session, &[0, 1, 2, 3, 4, 5]).await;
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&HlsAccessLeaseId(lease.to_string()),
|
||||
HlsAccessLeaseState::Pending,
|
||||
&StripConfig { mode: HlsStripMode::Segments, value: 0 },
|
||||
None,
|
||||
HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.expect("complete baseline is published to the playback lease");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let first = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
deliver(&app, &session, &first, &[1000]).await;
|
||||
tokio::time::timeout(WAIT, async {
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
loop {
|
||||
let snapshot = app
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(&HlsAccessLeaseId(lease.to_string()), &proxy_id, current_time_millis())
|
||||
.await
|
||||
.unwrap();
|
||||
if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(0) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("baseline body completion is recorded before recovery pressure is evaluated");
|
||||
let (binding, baseline, commits, proxy_id) = {
|
||||
let s = session.read().await;
|
||||
assert!(s.activity.last_authorized_media_at_ms.is_some());
|
||||
assert!(s.activity.last_delivered_media_at_ms.is_some());
|
||||
assert!(s.established_manifest_recovery_binding().is_some());
|
||||
(
|
||||
s.origin_account_binding.clone().unwrap(),
|
||||
s.origin_to_proxy.clone(),
|
||||
s.origin_manifest_commits(),
|
||||
s.proxy_session_id.clone(),
|
||||
)
|
||||
};
|
||||
assert_eq!(binding.account_name.as_ref(), "account-a");
|
||||
eprintln!(
|
||||
"{mode:?} baseline account={} binding_generation={} pinned_at_ms={} proxy_session={}",
|
||||
binding.account_name, binding.generation, binding.pinned_at_ms, proxy_id.0
|
||||
);
|
||||
session.write().await.origin_control.path_condition = tuliprox_hls::HlsOriginPathCondition::HardFetchFailure;
|
||||
let directive =
|
||||
match tuliprox_hls::api::hls_manifest_acceptance_directive_for_session(&app.hls_ctx(), &session, &proxy_id)
|
||||
.await
|
||||
{
|
||||
tuliprox_hls::api::HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive,
|
||||
other => panic!("recovery pressure must evaluate: {other:?}"),
|
||||
};
|
||||
eprintln!("app baseline directive={directive:?}");
|
||||
assert!(directive.recovery_pressure_guard.is_some(), "directive has current lease/session evidence");
|
||||
assert_eq!(directive.trigger, tuliprox_hls::HlsManifestAcceptanceTrigger::RecoveryRequired);
|
||||
let mut io = super::provider_admission::test_hls_origin_io_context(&app);
|
||||
io.account_runtime = session.read().await.origin_account_runtime.clone();
|
||||
let snapshot = HlsOriginAccountSnapshot::from_session(&*session.read().await);
|
||||
let mut request = hls_origin_refresh_request(
|
||||
&app,
|
||||
&session,
|
||||
HlsPreparedOriginAccount { entry: LiveHlsOriginEntry::parse(&url).unwrap(), snapshot },
|
||||
HlsOriginRefreshOptions {
|
||||
headers: HeaderMap::new(),
|
||||
origin_io: io,
|
||||
manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed,
|
||||
acceptance_directive: directive,
|
||||
access_lease_id: Some(HlsAccessLeaseId(lease.to_string())),
|
||||
rewrite_secret: app.get_encrypt_secret().to_vec(),
|
||||
now_ms: current_time_millis(),
|
||||
},
|
||||
);
|
||||
request.now_ms = session.read().await.origin_refresh.next_fetch_allowed_at_ms.max(current_time_millis());
|
||||
control.recovering.store(true, Ordering::Release);
|
||||
let refresh = tokio::spawn(trigger_origin_refresh_sync(request));
|
||||
tokio::time::timeout(WAIT, control.retry_started.notified()).await.expect("reduced HTTP attempt 13 starts");
|
||||
let episode_before = {
|
||||
let s = session.read().await;
|
||||
assert_eq!(s.origin_to_proxy, baseline);
|
||||
assert_eq!(s.origin_manifest_commits(), commits, "first actual conflict commit did not mutate the timeline");
|
||||
let episode = s.origin_control.acceptance_episode.clone().unwrap();
|
||||
assert_eq!(episode.full_bursts_completed, 1);
|
||||
assert!(episode.exhaustion_reason().is_some());
|
||||
assert!(episode.deterministic_conflict_receipt().is_none());
|
||||
eprintln!(
|
||||
"{mode:?} before response 13: generation={} full_bursts={} reason={:?} timing={:?}",
|
||||
episode.generation.0,
|
||||
episode.full_bursts_completed,
|
||||
episode.exhaustion_reason(),
|
||||
episode.timing()
|
||||
);
|
||||
episode
|
||||
};
|
||||
let prepare_gate = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test();
|
||||
control.release_retry.notify_one();
|
||||
tokio::time::timeout(WAIT, prepare_gate.entered.notified())
|
||||
.await
|
||||
.expect("repaired reduced retry commits and schedules media");
|
||||
let scheduled = {
|
||||
let s = session.read().await;
|
||||
let mut resources = s
|
||||
.segment_prefetch_queue
|
||||
.proxy_seqs()
|
||||
.iter()
|
||||
.map(|seq| {
|
||||
let entry = &s.segments[seq];
|
||||
(
|
||||
*seq,
|
||||
entry.origin_key.host_local_sequence,
|
||||
entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
resources.sort();
|
||||
assert_eq!(resources.iter().map(|(_, seq, _)| *seq).collect::<Vec<_>>(), vec![1006, 1007, 1008]);
|
||||
assert_eq!(s.origin_manifest_commits(), commits + 1);
|
||||
assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation);
|
||||
assert_eq!(s.origin_control.acceptance_generation, episode_before.generation);
|
||||
assert!(
|
||||
s.origin_control.acceptance_episode.is_none(),
|
||||
"successful refresh bookkeeping consumes the completed episode"
|
||||
);
|
||||
eprintln!("{mode:?} committed generation={} scheduled={resources:?}", episode_before.generation.0);
|
||||
resources
|
||||
};
|
||||
prepare_gate.prepare.notify_one();
|
||||
tokio::time::timeout(WAIT, prepare_gate.prepared.notified()).await.unwrap();
|
||||
prepare_gate.proceed.notify_one();
|
||||
let seqs = scheduled.iter().map(|(seq, _, _)| *seq).collect::<Vec<_>>();
|
||||
ready(&app, &session, &seqs).await;
|
||||
tokio::time::timeout(WAIT, refresh).await.unwrap().unwrap();
|
||||
assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13);
|
||||
{
|
||||
let requests = origin.requests.lock().await;
|
||||
for (_, _, url) in &scheduled {
|
||||
let path = reqwest::Url::parse(url).unwrap().path().to_string();
|
||||
let count = requests.iter().filter(|request| request.starts_with(&format!("GET {path} "))).count();
|
||||
assert!(count > 0, "queued resource was fetched before any client demand");
|
||||
eprintln!("{mode:?} autonomous origin GET {path} requests={count} status=Ready");
|
||||
}
|
||||
assert!(requests.iter().all(|request| !request.contains("account-user")), "B remains unused");
|
||||
}
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&HlsAccessLeaseId(lease.to_string()),
|
||||
HlsAccessLeaseState::Activated,
|
||||
&StripConfig { mode: HlsStripMode::Segments, value: 0 },
|
||||
None,
|
||||
HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.expect("repaired live manifest is published");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
deliver(&app, &session, &body, &[1006, 1007, 1008]).await;
|
||||
let s = session.read().await;
|
||||
assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation);
|
||||
assert_eq!(s.origin_account_binding.as_ref().unwrap().account_name, binding.account_name);
|
||||
assert_eq!(s.origin_account_binding.as_ref().unwrap().pinned_at_ms, binding.pinned_at_ms);
|
||||
assert_eq!(s.origin_account_binding.as_ref().unwrap().provider_binding_tag, binding.provider_binding_tag);
|
||||
assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13);
|
||||
eprintln!(
|
||||
"{mode:?} complete: recovery_requests=13 account={} binding_generation={} origin_work=0 singleflight=false",
|
||||
binding.account_name, binding.generation
|
||||
);
|
||||
assert_eq!(s.proxy_session_id, proxy_id);
|
||||
assert!(!s.origin_refresh.in_flight);
|
||||
assert_eq!(s.active_segment_fetches, 0);
|
||||
assert_eq!(s.activity.active_origin_work_count.load(Ordering::Acquire), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_reduced_retry_delivers_repaired_origin_media_optimistic() {
|
||||
repaired_reduced_retry(HlsStartupMode::Optimistic).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_reduced_retry_delivers_repaired_origin_media_conservative() {
|
||||
repaired_reduced_retry(HlsStartupMode::Conservative).await;
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
use super::{
|
||||
manifest_entry_freshness::open_manifest_for_new_lease,
|
||||
origin_account_rebind::{enable_isolated_rebind_cache, resume_origin, resume_origin_media},
|
||||
*,
|
||||
};
|
||||
use shared::model::{HlsStartupConfigDto, HlsStartupMode};
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use tuliprox_core::utils::current_time_millis;
|
||||
|
||||
const PAUSE_MS: u64 = 130_000;
|
||||
const WAIT: Duration = Duration::from_secs(5);
|
||||
|
||||
async fn settle_origin(session: &HlsSessionHandle) {
|
||||
tokio::time::timeout(WAIT, async {
|
||||
loop {
|
||||
let s = session.read().await;
|
||||
if !s.origin_refresh.in_flight
|
||||
&& s.active_segment_fetches == 0
|
||||
&& s.activity.active_origin_work_count.load(Ordering::Acquire) == 0
|
||||
{
|
||||
break;
|
||||
}
|
||||
drop(s);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("origin work settles before playback is stopped");
|
||||
}
|
||||
|
||||
/// Model wall-clock absence without erasing actual media delivery, or relying on
|
||||
/// Tokio's clock (account preparation and HTTP serving also read `SystemTime`).
|
||||
async fn pause_playback(app: &Arc<AppState>, session: &HlsSessionHandle, lease: &str) {
|
||||
settle_origin(session).await;
|
||||
assert!(app
|
||||
.hls
|
||||
.proxy
|
||||
.segment_worker_pool()
|
||||
.access_leases()
|
||||
.write()
|
||||
.await
|
||||
.remove_access_lease(&HlsAccessLeaseId(lease.to_string()))
|
||||
.is_some());
|
||||
app.active_users.release_session_streams_and_counted_reservation("hls-user", "hls-session-token").await;
|
||||
let now = current_time_millis();
|
||||
let proxy_id = {
|
||||
let mut s = session.write().await;
|
||||
let age = |time: &mut Option<u64>| *time = time.map(|time| time.saturating_sub(PAUSE_MS));
|
||||
age(&mut s.activity.last_authorized_manifest_at_ms);
|
||||
age(&mut s.activity.last_authorized_media_at_ms);
|
||||
age(&mut s.activity.last_delivered_media_at_ms);
|
||||
s.last_client_access_at_ms = s.last_client_access_at_ms.saturating_sub(PAUSE_MS);
|
||||
let binding = s.origin_account_binding.as_mut().unwrap();
|
||||
binding.pinned_at_ms = binding.pinned_at_ms.saturating_sub(PAUSE_MS);
|
||||
age(&mut binding.last_origin_io_at_ms);
|
||||
age(&mut binding.last_reservation_refresh_at_ms);
|
||||
age(&mut s.origin_refresh.last_fetch_started_at_ms);
|
||||
age(&mut s.origin_refresh.last_fetch_finished_at_ms);
|
||||
age(&mut s.origin_refresh.last_success_at_ms);
|
||||
age(&mut s.origin_refresh.last_error_at_ms);
|
||||
s.origin_refresh.next_fetch_allowed_at_ms = s.origin_refresh.next_fetch_allowed_at_ms.saturating_sub(PAUSE_MS);
|
||||
if let Some(rendered) = s.last_rendered_manifest.as_mut() {
|
||||
rendered.rendered_at_ms = rendered.rendered_at_ms.saturating_sub(PAUSE_MS);
|
||||
rendered.valid_until_ms = rendered.valid_until_ms.saturating_sub(PAUSE_MS);
|
||||
}
|
||||
assert!(s.activity.last_delivered_media_at_ms.is_some_and(|last| now - last >= PAUSE_MS));
|
||||
assert!(!s.can_expire_idle_session(now, 300_000), "retained session is inside its 300-second timeout");
|
||||
s.proxy_session_id.clone()
|
||||
};
|
||||
app.hls
|
||||
.proxy
|
||||
.sync_session_access_lease_count_and_detach_if_needed(
|
||||
&app.active_users,
|
||||
&app.active_provider,
|
||||
session,
|
||||
&proxy_id,
|
||||
now,
|
||||
)
|
||||
.await;
|
||||
super::super::detach_unprotected_hls_origin_account_bindings(app, now).await;
|
||||
assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached());
|
||||
assert_eq!(app.active_provider.get_provider_connections_count(), 0, "pause releases provider slots");
|
||||
for account in ["account-a", "account-b"] {
|
||||
assert_eq!(
|
||||
app.active_provider.provider_lease_usage(&Arc::from(account)).total(),
|
||||
0,
|
||||
"no logical slot remains for {account}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn segment_uris(manifest: &str) -> Vec<&str> {
|
||||
assert!(manifest.starts_with("#EXTM3U"), "live playlist: {manifest}");
|
||||
assert!(!manifest.contains("/cvs/hls/"), "replacement video must never satisfy resume: {manifest}");
|
||||
assert!(!manifest.contains("#EXT-X-ENDLIST"), "resume remains live");
|
||||
let uris = manifest.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::<Vec<_>>();
|
||||
assert!(!uris.is_empty(), "live media playlist must contain media");
|
||||
uris
|
||||
}
|
||||
|
||||
async fn deliver_origin_segment(app: &Arc<AppState>, session: &HlsSessionHandle, manifest: &str, head: u64) {
|
||||
let uri = segment_uris(manifest)[0];
|
||||
let seq = uri.rsplit('/').next().unwrap().trim_end_matches(".ts").parse::<u64>().unwrap();
|
||||
let origin_url = {
|
||||
let s = session.read().await;
|
||||
let segment = &s.segments[&seq];
|
||||
assert!((head..head + 6).contains(&segment.origin_key.host_local_sequence), "current origin window");
|
||||
segment.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone()
|
||||
};
|
||||
let path = reqwest::Url::parse(&origin_url).unwrap().path().to_string();
|
||||
let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_body(response).await;
|
||||
assert_eq!(body.as_ref(), resume_origin_media(&path), "client receives the exact current origin object");
|
||||
let s = session.read().await;
|
||||
assert!(s.activity.last_authorized_media_at_ms.is_some());
|
||||
assert!(s.activity.last_delivered_media_at_ms.is_some(), "fully consume the Tuliprox response before pausing");
|
||||
eprintln!(
|
||||
"delivered origin={path} bytes={} authorized={:?} delivered={:?}",
|
||||
body.len(),
|
||||
s.activity.last_authorized_media_at_ms,
|
||||
s.activity.last_delivered_media_at_ms
|
||||
);
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_lines, reason = "Keep the ordered pause/resume scenario and its assertions together.")]
|
||||
async fn same_account_resume(mode: HlsStartupMode, resumes: usize) {
|
||||
let head = Arc::new(AtomicU64::new(1113));
|
||||
let origin = resume_origin(Arc::clone(&head)).await;
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let mut input = overlap_provider_input();
|
||||
input.name = Arc::from("account-a");
|
||||
input.priority = 0;
|
||||
input.url.clone_from(&origin.base_url);
|
||||
let b = &mut input.aliases.as_mut().unwrap()[0];
|
||||
b.name = Arc::from("account-b");
|
||||
b.priority = 10;
|
||||
b.url.clone_from(&origin.base_url);
|
||||
let app = test_app_state_with_hls_proxy_and_inputs(
|
||||
Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)),
|
||||
vec![Arc::new(input.clone())],
|
||||
);
|
||||
let available_b = HlsOriginAccountBinding::new(
|
||||
Arc::clone(&input.name),
|
||||
Arc::from("account-b"),
|
||||
&ProxySessionId("availability-check".into()),
|
||||
current_time_millis(),
|
||||
);
|
||||
assert_eq!(
|
||||
tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b),
|
||||
tuliprox_hls::api::HlsOriginAccountStatus::Known
|
||||
);
|
||||
enable_isolated_rebind_cache(&app, cache.path());
|
||||
let mut config = app.app_config.config.load_full().as_ref().clone();
|
||||
config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap().startup =
|
||||
(&HlsStartupConfigDto { mode, ..HlsStartupConfigDto::default() }).into();
|
||||
app.app_config.config.store(Arc::new(config));
|
||||
enable_channel_unavailable_custom_response(&app);
|
||||
app.hls.proxy.update_config(&app.app_config).await;
|
||||
assert_eq!(app.hls.proxy.session_idle_timeout_ms(), 300_000);
|
||||
create_active_hls_user_session(&app).await;
|
||||
let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url);
|
||||
let key = super::super::build_hls_origin_source(&input, "12345").session_key();
|
||||
let mut lease = "resume-initial".to_string();
|
||||
let first = open_manifest_for_new_lease(&app, &input, &url, &lease).await;
|
||||
let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap();
|
||||
deliver_origin_segment(&app, &session, &first, 1113).await;
|
||||
settle_origin(&session).await;
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
|
||||
for cycle in 0..resumes {
|
||||
let old = session.read().await.origin_account_binding.clone().unwrap();
|
||||
assert_eq!(old.account_name.as_ref(), "account-a");
|
||||
pause_playback(&app, &session, &lease).await;
|
||||
let current_head = 1126 + u64::try_from(cycle).unwrap() * 13;
|
||||
head.store(current_head, Ordering::Release);
|
||||
lease = format!("resume-{cycle}");
|
||||
create_active_hls_user_session(&app).await;
|
||||
let boundary = origin.requests.lock().await.len();
|
||||
let started_at = current_time_millis();
|
||||
let barrier = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test();
|
||||
let (task_app, task_input, task_url, task_lease) =
|
||||
(Arc::clone(&app), input.clone(), url.clone(), lease.clone());
|
||||
let opening =
|
||||
tokio::spawn(
|
||||
async move { open_manifest_for_new_lease(&task_app, &task_input, &task_url, &task_lease).await },
|
||||
);
|
||||
tokio::time::timeout(WAIT, barrier.entered.notified()).await.expect("manifest commit reaches scheduler");
|
||||
let retained = app.hls.proxy.sessions().get_by_key(&key).await.unwrap();
|
||||
assert!(Arc::ptr_eq(&session, &retained), "same shared session allocation");
|
||||
let (rebound, scheduled, delivered_before_prepare) = {
|
||||
let s = session.read().await;
|
||||
assert_eq!(s.key, key);
|
||||
assert_eq!(s.proxy_session_id, proxy_id);
|
||||
assert_eq!(s.target_duration, Some(11));
|
||||
let rebound = s.origin_account_binding.clone().unwrap();
|
||||
assert_eq!(rebound.account_name, old.account_name, "B is available; A is actually rebound");
|
||||
assert!(rebound.is_active());
|
||||
assert!(rebound.generation > old.generation);
|
||||
assert!(rebound.pinned_at_ms >= started_at);
|
||||
let mut scheduled = s
|
||||
.segment_prefetch_queue
|
||||
.proxy_seqs()
|
||||
.into_iter()
|
||||
.map(|seq| {
|
||||
let entry = &s.segments[&seq];
|
||||
assert!((current_head..current_head + 6).contains(&entry.origin_key.host_local_sequence));
|
||||
(seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
scheduled.sort();
|
||||
assert!(!scheduled.is_empty(), "capture jobs immediately after commit, before preparation");
|
||||
eprintln!("{mode:?} cycle={cycle} shared={key:?} proxy={proxy_id:?} account={} binding={}->{} pinned={} scheduled={scheduled:?}", rebound.account_name, old.generation, rebound.generation, rebound.pinned_at_ms);
|
||||
(rebound, scheduled, s.activity.last_delivered_media_at_ms)
|
||||
};
|
||||
barrier.prepare.notify_one();
|
||||
tokio::time::timeout(WAIT, barrier.prepared.notified()).await.expect("resource preparation completes");
|
||||
eprintln!(
|
||||
"requests through prepare={:?}",
|
||||
origin.requests.lock().await[boundary..]
|
||||
.iter()
|
||||
.map(|request| request.lines().next().unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
{
|
||||
let s = session.read().await;
|
||||
eprintln!(
|
||||
"after prepare: binding={:?} queue={:?} states={:?}",
|
||||
s.origin_account_binding,
|
||||
s.segment_prefetch_queue.proxy_seqs(),
|
||||
scheduled.iter().map(|(seq, _)| (*seq, &s.segments[seq].status)).collect::<Vec<_>>()
|
||||
);
|
||||
assert!(
|
||||
scheduled.iter().all(|(seq, _)| matches!(
|
||||
s.segments[seq].status,
|
||||
SegmentCacheStatus::Queued { .. }
|
||||
| SegmentCacheStatus::Fetching { .. }
|
||||
| SegmentCacheStatus::Ready { .. }
|
||||
)),
|
||||
"manifest segment jobs were lost during resource preparation"
|
||||
);
|
||||
let current = s.origin_account_binding.as_ref().unwrap();
|
||||
assert_eq!(current.generation, rebound.generation, "fresh binding stays stable into media work");
|
||||
assert_eq!(current.pinned_at_ms, rebound.pinned_at_ms);
|
||||
assert_eq!(
|
||||
s.activity.last_delivered_media_at_ms, delivered_before_prepare,
|
||||
"manifest fetch is not media delivery"
|
||||
);
|
||||
}
|
||||
barrier.proceed.notify_one();
|
||||
// No demand fetch or test-side requeue: every original job must progress autonomously.
|
||||
tokio::time::timeout(WAIT, async {
|
||||
loop {
|
||||
let s = session.read().await;
|
||||
if scheduled.iter().all(|(seq, _)| matches!(s.segments[seq].status, SegmentCacheStatus::Ready { .. })) {
|
||||
break;
|
||||
}
|
||||
drop(s);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all originally scheduled resources reach Ready");
|
||||
let manifest = tokio::time::timeout(WAIT, opening).await.unwrap().unwrap();
|
||||
let requests = origin.requests.lock().await[boundary..]
|
||||
.iter()
|
||||
.map(|request| request.lines().next().unwrap().to_string())
|
||||
.collect::<Vec<_>>();
|
||||
eprintln!("{mode:?} cycle={cycle} origin requests={requests:?}");
|
||||
for (_, url) in &scheduled {
|
||||
let path = reqwest::Url::parse(url).unwrap().path().to_string();
|
||||
assert!(
|
||||
requests.iter().any(|line| line == &format!("GET {path} HTTP/1.1")),
|
||||
"scheduled origin request: {path}"
|
||||
);
|
||||
}
|
||||
assert!(requests.iter().all(|line| !line.contains("account-user")), "B remains unused");
|
||||
deliver_origin_segment(&app, &session, &manifest, current_head).await;
|
||||
assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, rebound.generation);
|
||||
}
|
||||
pause_playback(&app, &session, &lease).await;
|
||||
assert!(origin.requests.lock().await.iter().all(|request| !request.contains("account-user")));
|
||||
assert_eq!(app.active_provider.get_provider_connections_count(), 0, "no provider slot leak after repeated pauses");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_same_account_resume_keeps_scheduled_segments() {
|
||||
same_account_resume(HlsStartupMode::Optimistic, 2).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_same_account_resume_keeps_scheduled_segments_conservative() {
|
||||
same_account_resume(HlsStartupMode::Conservative, 2).await;
|
||||
}
|
||||
@@ -346,6 +346,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_segment_entry(
|
||||
host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX),
|
||||
},
|
||||
proxy_seq,
|
||||
media_start_ms: proxy_seq * 4_000,
|
||||
duration_ms: 4_000,
|
||||
proxy_file_ext: "ts".to_string(),
|
||||
content_type: "video/mp2t".to_string(),
|
||||
|
||||
@@ -94,6 +94,8 @@ pub(in crate::api::endpoints::hls_api::tests) fn store_normal_manifest_body_rang
|
||||
SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms },
|
||||
);
|
||||
entry.duration_ms = 10_000;
|
||||
entry.media_start_ms = proxy_seq * 10_000;
|
||||
session.next_media_start_ms = session.next_media_start_ms.max(entry.media_start_ms + entry.duration_ms);
|
||||
session.segments.insert(proxy_seq, entry);
|
||||
}
|
||||
session.advance_media_readiness_generation();
|
||||
|
||||
@@ -225,6 +225,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_runtime_policy_t
|
||||
HlsLeasePlaybackMode::Live => "live",
|
||||
HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail",
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable",
|
||||
HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response",
|
||||
HlsLeasePlaybackMode::Ended => "ended",
|
||||
});
|
||||
panic!(
|
||||
|
||||
@@ -265,7 +265,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn hls_session_last_media_at
|
||||
pub(in crate::api::endpoints::hls_api::tests) async fn assert_no_hls_cache_stream_registered(
|
||||
app_state: &Arc<AppState>,
|
||||
) {
|
||||
assert!(app_state.active_users.active_streams().await.is_empty());
|
||||
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
|
||||
}
|
||||
|
||||
pub(in crate::api::endpoints::hls_api::tests) async fn access_lease_session_token(
|
||||
|
||||
@@ -230,11 +230,16 @@ pub(in crate::api::endpoints::hls_api::tests) async fn run_recovery_outage(
|
||||
let episode = session.origin_control.acceptance_episode.as_ref().expect("bounded acceptance evidence");
|
||||
assert!(episode.full_burst_completed);
|
||||
assert_eq!(episode.completed_burst_candidates, plan.total_candidates());
|
||||
last_episode_generation = Some(episode.generation.0);
|
||||
if let Some(generation) = last_episode_generation {
|
||||
assert_eq!(episode.generation.0, generation, "unchanged failures share the initial full-burst receipt");
|
||||
} else {
|
||||
last_episode_generation = Some(episode.generation.0);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
fixture.origin.manifest_request_count().saturating_sub(requests_before)
|
||||
>= plan.total_candidates().saturating_mul(3)
|
||||
assert_eq!(
|
||||
fixture.origin.manifest_request_count().saturating_sub(requests_before),
|
||||
plan.total_candidates() + fixture.refresh.retry_policy.attempt_count().saturating_sub(1) + 2,
|
||||
"one configured full burst, its bounded reduced retries, then one sample per due refresh"
|
||||
);
|
||||
let uri = format!("/hls/shared/live/{}/{}/000003.ts", fixture.proxy_session_id.0, fixture.lease_id.0);
|
||||
let cached = get_response(Arc::clone(&fixture.app_state), &uri, None).await;
|
||||
@@ -794,9 +799,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr
|
||||
session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure;
|
||||
let transition_buffer_seq = fixture.base_manifest.last_proxy_seq.saturating_add(1);
|
||||
let commit_guard_seq = transition_buffer_seq.saturating_add(1);
|
||||
let base_tail = session.segments.get(&fixture.base_manifest.last_proxy_seq).expect("live base tail");
|
||||
let mut media_start_ms = base_tail.media_start_ms.saturating_add(base_tail.duration_ms);
|
||||
for segment in
|
||||
session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq)
|
||||
{
|
||||
// Shortened test buffers must remain contiguous on the session media clock.
|
||||
segment.media_start_ms = media_start_ms;
|
||||
if segment.proxy_seq == transition_buffer_seq {
|
||||
segment.duration_ms = 1_000;
|
||||
} else if segment.proxy_seq == commit_guard_seq {
|
||||
@@ -804,6 +813,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr
|
||||
} else {
|
||||
segment.status = SegmentCacheStatus::Expired;
|
||||
}
|
||||
media_start_ms = media_start_ms.saturating_add(segment.duration_ms);
|
||||
}
|
||||
session.advance_media_readiness_generation();
|
||||
}
|
||||
@@ -915,7 +925,11 @@ pub(in crate::api::endpoints::hls_api::tests) async fn commit_prepared_terminal_
|
||||
.playback_cursor
|
||||
.first_segment_completed_at_ms
|
||||
.expect("measured lease playback start")
|
||||
.saturating_add(fixture.base_manifest.playlist_duration_ms);
|
||||
// Enter the acquisition window while the live base and its key are still usable.
|
||||
.saturating_add(
|
||||
fixture.base_manifest.playlist_duration_ms.saturating_sub(fixture.base_manifest.target_duration_ms),
|
||||
);
|
||||
|
||||
let first = commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
&fixture.app_state.hls_ctx(),
|
||||
&fixture.session,
|
||||
@@ -1309,6 +1323,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn assert_publication_late_t
|
||||
assert!(body.contains("#EXT-X-ENDLIST"));
|
||||
assert!(body.contains("/terminal/"));
|
||||
}
|
||||
HlsLeasePlaybackMode::TerminalCustomResponse(_) => {
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest utf8");
|
||||
assert!(body.contains("#EXT-X-ENDLIST"));
|
||||
assert!(body.contains("/cvs/hls/"));
|
||||
assert!(!body.contains("/hls/shared/live/"));
|
||||
}
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { .. } => {
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
@@ -1352,6 +1373,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn regression_origin_refresh_reque
|
||||
transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(),
|
||||
manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed,
|
||||
fresh_manifest_requirement_generation: None,
|
||||
refresh_identity: None,
|
||||
acceptance_directive: HlsManifestAcceptanceDirective::none(),
|
||||
access_lease_id,
|
||||
disabled_headers: None,
|
||||
|
||||
@@ -86,7 +86,7 @@ async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() {
|
||||
async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhausted_recovery() {
|
||||
let fixture = prepared_terminal_cutover_fixture().await;
|
||||
let directive = terminal_cutover_acceptance_directive(&fixture).await;
|
||||
let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await;
|
||||
@@ -96,7 +96,7 @@ async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_splice() {
|
||||
async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splice_deadline() {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir");
|
||||
let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)));
|
||||
enable_channel_unavailable_custom_response(&app_state);
|
||||
@@ -147,16 +147,30 @@ async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_s
|
||||
|
||||
let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(!response.headers().contains_key(header::LOCATION));
|
||||
assert!(response_body(response).await.is_empty());
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest");
|
||||
assert!(body.ends_with("#EXT-X-ENDLIST\n"));
|
||||
assert!(body.contains("/cvs/hls/"));
|
||||
assert!(!body.contains("#EXT-X-MAP"));
|
||||
assert!(!body.contains("/hls/shared/live/"));
|
||||
let lease = app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis())
|
||||
.await
|
||||
.expect("failed-closed lease snapshot");
|
||||
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
|
||||
.expect("late custom lease snapshot");
|
||||
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
|
||||
let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await;
|
||||
assert_eq!(repeated.status(), StatusCode::OK);
|
||||
assert_eq!(response_body(repeated).await, body.as_bytes());
|
||||
let repeated_lease = app_state
|
||||
.hls
|
||||
.proxy
|
||||
.access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis())
|
||||
.await
|
||||
.expect("stable terminal custom lease");
|
||||
assert_eq!(repeated_lease.playback_mode, lease.playback_mode);
|
||||
let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0);
|
||||
let terminal_response = get_response(app_state, &terminal_uri, None).await;
|
||||
assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND);
|
||||
|
||||
@@ -44,6 +44,7 @@ pub(super) async fn hls_manifest_terminal_preflight(
|
||||
) -> HlsManifestTerminalPreflight {
|
||||
match &lease.playback_mode {
|
||||
HlsLeasePlaybackMode::TerminalTail(_)
|
||||
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
|
||||
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
|
||||
| HlsLeasePlaybackMode::Ended => return HlsManifestTerminalPreflight::ServeCommittedPlayback,
|
||||
HlsLeasePlaybackMode::Live => {}
|
||||
@@ -183,6 +184,13 @@ pub(super) fn hls_terminal_playback_response(
|
||||
};
|
||||
Some(hls_response(body.to_owned()))
|
||||
}
|
||||
HlsLeasePlaybackMode::TerminalCustomResponse(plan) => {
|
||||
Some(if plan.matches_route(proxy_session_id, access_lease_id) {
|
||||
hls_response(plan.manifest_body().to_owned())
|
||||
} else {
|
||||
StatusCode::NOT_FOUND.into_response()
|
||||
})
|
||||
}
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { .. } => Some(StatusCode::SERVICE_UNAVAILABLE.into_response()),
|
||||
HlsLeasePlaybackMode::Live => None,
|
||||
HlsLeasePlaybackMode::Ended => Some(StatusCode::NOT_FOUND.into_response()),
|
||||
|
||||
@@ -1186,11 +1186,11 @@ mod tests {
|
||||
|
||||
assert_eq!(api_proxy.server.len(), 1);
|
||||
assert_eq!(api_proxy.server[0].name, "main");
|
||||
assert!(api_proxy.server[0].protocol.is_empty());
|
||||
assert!(api_proxy.server[0].host.is_empty());
|
||||
assert_eq!(api_proxy.server[0].protocol.len(), 0);
|
||||
assert_eq!(api_proxy.server[0].host.len(), 0);
|
||||
assert!(api_proxy.server[0].port.is_none());
|
||||
assert!(api_proxy.server[0].timezone.is_empty());
|
||||
assert!(api_proxy.server[0].message.is_empty());
|
||||
assert_eq!(api_proxy.server[0].timezone.len(), 0);
|
||||
assert_eq!(api_proxy.server[0].message.len(), 0);
|
||||
assert!(api_proxy.server[0].path.is_none());
|
||||
assert_eq!(api_proxy.user.len(), 1);
|
||||
assert!(!api_proxy.use_user_db);
|
||||
@@ -1206,8 +1206,8 @@ mod tests {
|
||||
|
||||
assert_eq!(api_proxy.server.len(), 1);
|
||||
assert_eq!(api_proxy.server[0].name, "main");
|
||||
assert!(api_proxy.server[0].host.is_empty());
|
||||
assert!(api_proxy.user.is_empty());
|
||||
assert_eq!(api_proxy.server[0].host.len(), 0);
|
||||
assert_eq!(api_proxy.user.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1217,8 +1217,8 @@ mod tests {
|
||||
|
||||
filter_api_proxy_by_permissions(&mut api_proxy, permissions);
|
||||
|
||||
assert!(api_proxy.server.is_empty());
|
||||
assert!(api_proxy.user.is_empty());
|
||||
assert_eq!(api_proxy.server.len(), 0);
|
||||
assert_eq!(api_proxy.user.len(), 0);
|
||||
assert!(!api_proxy.use_user_db);
|
||||
assert_eq!(api_proxy.auth_error_status, 403);
|
||||
}
|
||||
@@ -1270,20 +1270,20 @@ mod tests {
|
||||
filter_app_config_by_permissions(&mut app_config, Some(permissions));
|
||||
|
||||
assert_eq!(app_config.config.storage_dir, None);
|
||||
assert!(app_config.sources.inputs.is_empty());
|
||||
assert_eq!(app_config.sources.inputs.len(), 0);
|
||||
assert!(app_config.sources.provider.is_none());
|
||||
assert!(app_config.sources.templates.is_none());
|
||||
assert!(app_config.mappings.is_none());
|
||||
assert!(app_config.templates.is_none());
|
||||
assert_eq!(app_config.sources.sources.len(), 1);
|
||||
assert!(app_config.sources.sources[0].inputs.is_empty());
|
||||
assert_eq!(app_config.sources.sources[0].inputs.len(), 0);
|
||||
assert_eq!(app_config.sources.sources[0].targets.len(), 1);
|
||||
|
||||
let target = &app_config.sources.sources[0].targets[0];
|
||||
assert_eq!(target.name, "Default");
|
||||
assert_eq!(target.id, 1);
|
||||
assert_eq!(target.enabled, shared::model::ConfigTargetDto::default().enabled);
|
||||
assert!(target.output.is_empty());
|
||||
assert_eq!(target.output.len(), 0);
|
||||
assert!(target.curation.is_none(), "identity-only access must not disclose discovery credentials");
|
||||
assert!(target.filter.is_empty());
|
||||
assert!(target.options.is_none());
|
||||
@@ -1296,7 +1296,7 @@ mod tests {
|
||||
let api_proxy = app_config.api_proxy.expect("api proxy should remain present");
|
||||
assert_eq!(api_proxy.server.len(), 1);
|
||||
assert_eq!(api_proxy.server[0].name, "main");
|
||||
assert!(api_proxy.server[0].host.is_empty());
|
||||
assert_eq!(api_proxy.server[0].host.len(), 0);
|
||||
assert_eq!(api_proxy.user.len(), 1);
|
||||
}
|
||||
|
||||
|
||||
@@ -344,8 +344,8 @@ async fn manual_update_target_ids_preserve_named_legacy_and_empty_bulk_requests(
|
||||
assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]);
|
||||
} else {
|
||||
assert!(!queued.targets.enabled);
|
||||
assert!(queued.targets.targets.is_empty());
|
||||
assert!(queued.targets.target_names.is_empty());
|
||||
assert_eq!(queued.targets.targets.len(), 0);
|
||||
assert_eq!(queued.targets.target_names.len(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -582,7 +582,7 @@ async fn m3u_update_quality_playlist_update_status_reads_newest_timestamp_from_c
|
||||
assert_eq!(status.inputs[0].clusters[2].timestamp, 202);
|
||||
assert_eq!(status.inputs[1].input_id, 8);
|
||||
assert_eq!(status.inputs[1].last_update, None);
|
||||
assert!(status.inputs[1].clusters.is_empty());
|
||||
assert_eq!(status.inputs[1].clusters.len(), 0);
|
||||
assert_eq!(status.inputs[2].input_id, 9);
|
||||
assert_eq!(status.inputs[2].last_update, Some(404));
|
||||
assert_eq!(status.inputs[2].clusters.len(), 1, "Only the actually persisted M3U cluster is exposed");
|
||||
|
||||
@@ -50,7 +50,7 @@ async fn playlist_update_status_reload_returns_active_bus_facts_without_persisti
|
||||
PlaylistUpdateState::Failure,
|
||||
)));
|
||||
let Json(status) = super::super::playlist_update_status(State(app)).await.unwrap();
|
||||
assert!(status.active_updates.is_empty());
|
||||
assert_eq!(status.active_updates.len(), 0);
|
||||
assert_eq!(
|
||||
std::fs::read(path.join("status.json")).unwrap(),
|
||||
before,
|
||||
|
||||
@@ -747,7 +747,7 @@ fn m3u_timeshift_resolver_returns_resolved_url_for_flussonic_template() {
|
||||
.expect("M3U input handled by bridge");
|
||||
|
||||
assert_eq!(resolved.url, "http://provider.example/channel/video-1704067200-3600.m3u8");
|
||||
assert!(!resolved.discriminator.is_empty());
|
||||
assert_ne!(resolved.discriminator.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
//! Real-process regression: an origin account publication must reach every `WebUI` socket immediately.
|
||||
use axum::{
|
||||
extract::{OriginalUri, State},
|
||||
response::IntoResponse,
|
||||
Router,
|
||||
};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use shared::model::{ActiveUserConnectionChange, ProtocolMessage, SourcesConfigDto, StreamInfo, PROTOCOL_VERSION};
|
||||
use std::{
|
||||
fmt::Write,
|
||||
path::{Path, PathBuf},
|
||||
process::Stdio,
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
net::{TcpListener, TcpStream},
|
||||
sync::Notify,
|
||||
};
|
||||
use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream};
|
||||
|
||||
type Socket = WebSocketStream<MaybeTlsStream<TcpStream>>;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Origin {
|
||||
catalog: String,
|
||||
manifests: Arc<AtomicUsize>,
|
||||
segments: Arc<AtomicUsize>,
|
||||
proceed: Arc<Notify>,
|
||||
}
|
||||
|
||||
async fn origin(State(state): State<Origin>, OriginalUri(uri): OriginalUri) -> axum::response::Response {
|
||||
let path = uri.path();
|
||||
if path == "/__continue" {
|
||||
state.proceed.notify_one();
|
||||
return "continued".into_response();
|
||||
}
|
||||
if path.ends_with("catalog.m3u") {
|
||||
return ([("content-type", "audio/x-mpegurl")], state.catalog).into_response();
|
||||
}
|
||||
if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("m3u8")) {
|
||||
state.manifests.fetch_add(1, Ordering::AcqRel);
|
||||
return ([("content-type", "application/vnd.apple.mpegurl")], "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n100.ts\n#EXTINF:4,\n101.ts\n#EXTINF:4,\n102.ts\n#EXTINF:4,\n103.ts\n#EXTINF:4,\n104.ts\n#EXTINF:4,\n105.ts\n").into_response();
|
||||
}
|
||||
if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("ts")) {
|
||||
state.segments.fetch_add(1, Ordering::AcqRel);
|
||||
return (
|
||||
[("content-type", "video/mp2t")],
|
||||
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts"))
|
||||
.as_slice(),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
axum::http::StatusCode::NOT_FOUND.into_response()
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
directory: tempfile::TempDir,
|
||||
base: String,
|
||||
source: PathBuf,
|
||||
process: tokio::process::Child,
|
||||
origins: Vec<tokio::task::JoinHandle<()>>,
|
||||
b: Origin,
|
||||
proceed: Arc<Notify>,
|
||||
control: String,
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.process.start_kill();
|
||||
for origin in &self.origins {
|
||||
origin.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
async fn start() -> Self {
|
||||
let a = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let b = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let a_addr = format!("localhost:{}", a.local_addr().unwrap().port());
|
||||
let b_addr = b.local_addr().unwrap();
|
||||
let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = reserved.local_addr().unwrap().port();
|
||||
drop(reserved);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let root = directory.path();
|
||||
for name in ["web", "storage", "backup"] {
|
||||
std::fs::create_dir(root.join(name)).unwrap();
|
||||
}
|
||||
let frontend = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend/dist");
|
||||
let web = if frontend.join("index.html").exists() { frontend } else { root.join("web") };
|
||||
let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: true\n user_ui_enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n stream:\n hls_session_ttl_secs: 120\n hls_cache:\n cache_path: '{}'\n", web.display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display());
|
||||
let source_text = format!("inputs:\n - name: account-a\n type: m3u\n url: http://{a_addr}/catalog.m3u\n max_connections: 8\n aliases:\n - name: account-b\n url: http://{b_addr}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [account-a]\n targets:\n - name: handover\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n");
|
||||
let mut users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Handover test\nuser:\n - target: handover\n credentials:\n");
|
||||
for user in ["alice", "bob", "late"] {
|
||||
write!(users, " - username: {user}\n password: test-password\n proxy: reverse\n server: local\n status: Active\n").unwrap();
|
||||
}
|
||||
let config_path = root.join("config.yml");
|
||||
let source = root.join("source.yml");
|
||||
let proxy = root.join("api-proxy.yml");
|
||||
std::fs::write(&config_path, config).unwrap();
|
||||
std::fs::write(&source, source_text).unwrap();
|
||||
std::fs::write(&proxy, users).unwrap();
|
||||
let proceed = Arc::new(Notify::new());
|
||||
let catalog = format!("#EXTM3U\n#EXTINF:-1 tvg-id=\"handover\" group-title=\"Test\",Handover Channel\nhttp://{a_addr}/live/index.m3u8\n");
|
||||
let state = |catalog: String| Origin {
|
||||
catalog,
|
||||
manifests: Arc::new(AtomicUsize::new(0)),
|
||||
segments: Arc::new(AtomicUsize::new(0)),
|
||||
proceed: proceed.clone(),
|
||||
};
|
||||
let a_state = state(catalog.clone());
|
||||
let b_state = state(catalog.replace(&a_addr, &b_addr.to_string()));
|
||||
let mut origins = Vec::new();
|
||||
for (listener, state) in [(a, a_state), (b, b_state.clone())] {
|
||||
origins.push(tokio::spawn(async move {
|
||||
axum::serve(listener, Router::new().fallback(origin).with_state(state)).await.unwrap();
|
||||
}));
|
||||
}
|
||||
let log = std::fs::File::create(root.join("sut.log")).unwrap();
|
||||
let process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox"))
|
||||
.args(["-s", "-c"])
|
||||
.arg(config_path)
|
||||
.arg("-i")
|
||||
.arg(&source)
|
||||
.arg("-a")
|
||||
.arg(proxy)
|
||||
.current_dir(root)
|
||||
.stdout(Stdio::from(log.try_clone().unwrap()))
|
||||
.stderr(Stdio::from(log))
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.unwrap();
|
||||
Self {
|
||||
directory,
|
||||
base: format!("http://127.0.0.1:{port}"),
|
||||
source,
|
||||
process,
|
||||
origins,
|
||||
b: b_state,
|
||||
proceed,
|
||||
control: format!("http://{a_addr}/__continue"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn playback(&mut self, client: &reqwest::Client, username: &str) -> String {
|
||||
let url = format!("{}/m3u?username={username}&password=test-password", self.base);
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
assert!(
|
||||
self.process.try_wait().unwrap().is_none(),
|
||||
"SUT exited: {}",
|
||||
std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap()
|
||||
);
|
||||
if let Ok(response) = client.get(&url).send().await {
|
||||
if let Ok(body) = response.text().await {
|
||||
if let Some(url) = body.lines().find(|line| line.starts_with("http")) {
|
||||
return url.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"playlist unavailable: {}",
|
||||
std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap()
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_browser_handover(&self, viewer_clients: &[reqwest::Client], viewer_media: &[reqwest::Url]) {
|
||||
eprintln!(
|
||||
"BROWSER_READY web={} control={} fixture={}",
|
||||
self.base,
|
||||
self.control,
|
||||
self.directory.path().display()
|
||||
);
|
||||
// Keep the two manual test players running while the browser loads. Stop all
|
||||
// media requests before publishing B; the browser must then update from events alone.
|
||||
let mut playback = tokio::time::interval(Duration::from_secs(1));
|
||||
loop {
|
||||
tokio::select! {
|
||||
() = self.proceed.notified() => break,
|
||||
_ = playback.tick() => {
|
||||
for (client, media) in viewer_clients.iter().zip(viewer_media) {
|
||||
let response = client.get(media.clone()).send().await.unwrap();
|
||||
assert!(response.status().is_success());
|
||||
response.bytes().await.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn socket(base: &str) -> Socket {
|
||||
let (mut socket, _) = connect_async(format!("{}/ws", base.replace("http://", "ws://"))).await.unwrap();
|
||||
socket.send(Message::Binary(ProtocolMessage::Version(PROTOCOL_VERSION).to_bytes().unwrap())).await.unwrap();
|
||||
assert!(
|
||||
matches!(socket.next().await.unwrap().unwrap(), Message::Binary(bytes) if bytes.as_ref() == [PROTOCOL_VERSION])
|
||||
);
|
||||
socket
|
||||
}
|
||||
|
||||
async fn frame(socket: &mut Socket) -> ProtocolMessage {
|
||||
loop {
|
||||
let next = tokio::time::timeout(Duration::from_secs(15), socket.next())
|
||||
.await
|
||||
.expect("WebUI frame arrives without polling")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
if let Message::Binary(bytes) = next {
|
||||
return ProtocolMessage::from_bytes(bytes).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot(socket: &mut Socket) -> Vec<StreamInfo> {
|
||||
socket.send(Message::Binary(ProtocolMessage::StatusRequest(String::new()).to_bytes().unwrap())).await.unwrap();
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
|
||||
loop {
|
||||
if let ProtocolMessage::StatusResponse(status) =
|
||||
tokio::time::timeout_at(deadline, frame(socket)).await.expect("full status snapshot arrives")
|
||||
{
|
||||
return status.active_user_streams;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn changed_viewers(socket: &mut Socket, before: &[StreamInfo]) {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
|
||||
while seen.len() < before.len() {
|
||||
match tokio::time::timeout_at(deadline, frame(socket)).await.expect("all viewers receive Updated frames") {
|
||||
ProtocolMessage::ActiveUserResponse(ActiveUserConnectionChange::Updated(row))
|
||||
if row.provider.as_ref() == "account-b" =>
|
||||
{
|
||||
let mut expected = before
|
||||
.iter()
|
||||
.find(|old| old.uid == row.uid && old.addr == row.addr)
|
||||
.expect("same viewer row")
|
||||
.clone();
|
||||
expected.provider = Arc::from("account-b");
|
||||
assert_eq!(row, expected, "handover changes only the displayed provider");
|
||||
seen.insert(row.uid);
|
||||
}
|
||||
ProtocolMessage::StatusResponse(_) => panic!("handover must arrive as incremental Updated frames"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hls_origin_account_handover() {
|
||||
let mut fixture = Fixture::start().await;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap();
|
||||
let mut canonical = Vec::new();
|
||||
let mut viewer_clients = Vec::new();
|
||||
let mut viewer_media = Vec::new();
|
||||
for viewer in ["alice", "bob"] {
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap();
|
||||
let mut entry = fixture.playback(&client, viewer).await;
|
||||
let mut manifest = None;
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
let response = client.get(&entry).send().await.unwrap();
|
||||
if response.status().is_success() {
|
||||
let base = response.url().clone();
|
||||
let body = response.text().await.unwrap();
|
||||
let segments = body
|
||||
.lines()
|
||||
.filter(|line| !line.starts_with('#') && !line.is_empty())
|
||||
.map(|line| base.join(line).unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(segment) = segments.first() {
|
||||
if body.contains("#EXT-X-STREAM-INF:") {
|
||||
entry = segment.to_string();
|
||||
continue;
|
||||
}
|
||||
let media = client.get(segment.clone()).send().await.unwrap();
|
||||
assert!(media.status().is_success());
|
||||
assert!(
|
||||
!media.bytes().await.unwrap().is_empty(),
|
||||
"viewer actually consumes media before the segment gap"
|
||||
);
|
||||
viewer_media.push(segment.clone());
|
||||
manifest = Some(base);
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
canonical.push(manifest.expect("viewer has a canonical playable manifest"));
|
||||
viewer_clients.push(client);
|
||||
}
|
||||
let mut ui_a = socket(&fixture.base).await;
|
||||
let mut ui_b = socket(&fixture.base).await;
|
||||
let mut before = snapshot(&mut ui_a).await;
|
||||
assert_eq!(
|
||||
before.len(),
|
||||
2,
|
||||
"canonical={canonical:?} SUT log: {}",
|
||||
std::fs::read_to_string(fixture.directory.path().join("sut.log")).unwrap()
|
||||
);
|
||||
assert!(before.iter().all(|row| row.provider.as_ref() == "account-a"));
|
||||
let mut before_b = snapshot(&mut ui_b).await;
|
||||
assert_eq!(before_b.len(), 2);
|
||||
let manual = std::env::var_os("TULIPROX_HANDOVER_BROWSER_CHECK").is_some();
|
||||
if manual {
|
||||
fixture.wait_for_browser_handover(&viewer_clients, &viewer_media).await;
|
||||
// Playback above may advance timestamps; capture the exact pre-publication rows.
|
||||
before = snapshot(&mut ui_a).await;
|
||||
before_b = snapshot(&mut ui_b).await;
|
||||
assert_eq!(before.len(), 2);
|
||||
assert_eq!(before_b.len(), 2);
|
||||
}
|
||||
let mut sources: SourcesConfigDto =
|
||||
serde_saphyr::from_str(&std::fs::read_to_string(&fixture.source).unwrap()).unwrap();
|
||||
sources.inputs[0].account_disabled = true;
|
||||
let config = client.get(format!("{}/api/v1/config", fixture.base)).send().await.unwrap();
|
||||
let revision =
|
||||
config.headers().get(shared::utils::HEADER_CONFIG_SOURCES_REVISION).expect("source revision").clone();
|
||||
let response = client
|
||||
.post(format!("{}/api/v1/config/sources", fixture.base))
|
||||
.header(reqwest::header::IF_MATCH, revision)
|
||||
.json(&sources)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(response.status().is_success(), "source reload: {}", response.text().await.unwrap());
|
||||
// Only a manifest refresh triggers the handover; neither viewer requests another segment or status.
|
||||
let response = client.get(canonical[0].clone()).send().await.unwrap();
|
||||
assert!(response.status().is_success() || response.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE);
|
||||
changed_viewers(&mut ui_a, &before).await;
|
||||
changed_viewers(&mut ui_b, &before_b).await;
|
||||
eprintln!(
|
||||
"HANDOVER_EVENTS_DELIVERED replacement_manifest_requests={} replacement_segment_requests={}",
|
||||
fixture.b.manifests.load(Ordering::Acquire),
|
||||
fixture.b.segments.load(Ordering::Acquire)
|
||||
);
|
||||
let after = snapshot(&mut ui_a).await;
|
||||
assert_eq!(after.len(), 2);
|
||||
assert!(after.iter().all(|row| row.provider.as_ref() == "account-b"));
|
||||
if manual {
|
||||
eprintln!("BROWSER_SWITCHED web={} control={}", fixture.base, fixture.control);
|
||||
fixture.proceed.notified().await;
|
||||
}
|
||||
fixture.process.kill().await.unwrap();
|
||||
}
|
||||
@@ -793,7 +793,7 @@ mod tests {
|
||||
let response = local_response(StatusCode::OK, &[], encoded_binary.clone()).await;
|
||||
let mut untouched =
|
||||
decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await.expect("identity setup");
|
||||
assert!(untouched.decoded_from.is_empty());
|
||||
assert_eq!(untouched.decoded_from.len(), 0);
|
||||
assert_eq!(read_to_end_limited(&mut untouched.body, 1024).await.expect("untouched body"), encoded_binary);
|
||||
}
|
||||
|
||||
@@ -806,7 +806,7 @@ mod tests {
|
||||
.await
|
||||
.expect("identity setup");
|
||||
|
||||
assert!(decoded.decoded_from.is_empty());
|
||||
assert_eq!(decoded.decoded_from.len(), 0);
|
||||
assert_eq!(decoded.content_coding_observation(), None);
|
||||
assert!(!decoded.headers.contains_key(CONTENT_ENCODING));
|
||||
assert_eq!(read_to_end_limited(&mut decoded.body, 1024).await.expect("identity body"), encoded);
|
||||
|
||||
@@ -52,11 +52,11 @@ pub enum ResourceDestination {
|
||||
}
|
||||
|
||||
/// How long a verdict that allows a URL to be handed to a client is remembered.
|
||||
const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_secs(60);
|
||||
const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_mins(1);
|
||||
|
||||
/// How long a verdict that keeps a destination away from clients is remembered. Both classes fail
|
||||
/// closed, so they can be cached longer than the one that decides exposure.
|
||||
const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_secs(300);
|
||||
const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_mins(5);
|
||||
|
||||
/// How long a name that produced no answer within the resolution budget is remembered.
|
||||
///
|
||||
|
||||
@@ -108,7 +108,7 @@ async fn atomic_download_temp_files_are_unique_and_use_target_directory() {
|
||||
drop(second_output);
|
||||
drop(first_temp);
|
||||
drop(second_temp);
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -181,7 +181,7 @@ async fn local_epg_file_respects_max_download_bytes() {
|
||||
|
||||
assert!(err.to_string().contains("exceeds configured limit"));
|
||||
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -213,7 +213,7 @@ async fn file_url_epg_source_respects_streamed_size_limit() {
|
||||
|
||||
assert!(err.to_string().contains("exceeds configured limit"));
|
||||
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -243,7 +243,7 @@ async fn local_epg_replace_error_cleans_temp_file() {
|
||||
.expect_err("replacing a directory should fail");
|
||||
|
||||
assert!(persist.is_dir());
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -273,7 +273,7 @@ async fn local_epg_read_error_preserves_existing_cache_and_cleans_temp_file() {
|
||||
.expect_err("reading a directory as an EPG file should fail");
|
||||
|
||||
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -301,7 +301,7 @@ async fn unsupported_epg_url_scheme_is_rejected_without_network_access() {
|
||||
|
||||
assert!(err.to_string().contains("Unsupported EPG URL scheme 'ftp'"));
|
||||
assert!(!persist.exists());
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
@@ -370,7 +370,7 @@ async fn remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file() {
|
||||
|
||||
assert!(err.to_string().contains("exceeds configured limit"));
|
||||
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
server_handle.abort();
|
||||
}
|
||||
|
||||
@@ -409,7 +409,7 @@ async fn remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file()
|
||||
.expect_err("truncated response body should fail");
|
||||
|
||||
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
server_handle.abort();
|
||||
}
|
||||
|
||||
@@ -466,7 +466,7 @@ async fn parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomic
|
||||
assert_eq!(accepted.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(max_active.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(tokio::fs::read(&persist).await.expect("read refreshed cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n");
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
server_handle.abort();
|
||||
}
|
||||
|
||||
|
||||
@@ -37,5 +37,5 @@ async fn file_url_epg_source_is_copied_to_persist_path() {
|
||||
tokio::fs::read_to_string(&persist).await.expect("persisted content"),
|
||||
"BEGIN:VCALENDAR\nEND:VCALENDAR\n"
|
||||
);
|
||||
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
|
||||
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ async fn empty_and_no_match_are_complete_but_disabled_sources_are_not_configured
|
||||
let CurationRunOutcome::Complete(result) = run(&config(None), &catalog(), &server).await else {
|
||||
panic!("valid empty or no-match is complete")
|
||||
};
|
||||
assert!(result.memberships.is_empty());
|
||||
assert_eq!(result.memberships.len(), 0);
|
||||
assert_eq!(result.selectors.len(), 2);
|
||||
}
|
||||
let server = TestServer::new(http_response(200, PAGE)).await;
|
||||
|
||||
@@ -699,7 +699,7 @@ mod tests {
|
||||
panic!("an empty source is complete")
|
||||
};
|
||||
assert_eq!(reference_count, 0);
|
||||
assert!(memberships.is_empty());
|
||||
assert_eq!(memberships.len(), 0);
|
||||
|
||||
let no_match = vec![reference(CurationMediaKind::Movie, "Missing", None, Some(8), Some(1))];
|
||||
let SelectorOutcome::Complete { reference_count, memberships, .. } =
|
||||
@@ -708,7 +708,7 @@ mod tests {
|
||||
panic!("a complete source with no local match is complete")
|
||||
};
|
||||
assert_eq!(reference_count, 1);
|
||||
assert!(memberships.is_empty());
|
||||
assert_eq!(memberships.len(), 0);
|
||||
|
||||
let matched = vec![reference(CurationMediaKind::Movie, "Matched", None, Some(7), Some(1))];
|
||||
let SelectorOutcome::Complete { reference_count, memberships, .. } =
|
||||
|
||||
@@ -36,7 +36,7 @@ impl TestServer {
|
||||
}
|
||||
|
||||
pub(crate) async fn serve(responses: Vec<Vec<u8>>, delay: Duration, send_headers: bool) -> Self {
|
||||
assert!(!responses.is_empty());
|
||||
assert_ne!(responses.len(), 0);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/", listener.local_addr().unwrap());
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
@@ -136,7 +136,7 @@ async fn tmdb_empty_is_valid_only_at_the_start_with_coherent_totals() {
|
||||
for pages in [0, 1] {
|
||||
let body = json!({"page":1,"total_pages":pages,"total_results":0,"results":[]}).to_string();
|
||||
let server = TestServer::new(http_response(200, &body)).await;
|
||||
assert!(client(&server).fetch(&selector(100)).await.unwrap().is_empty());
|
||||
assert_eq!(client(&server).fetch(&selector(100)).await.unwrap().len(), 0);
|
||||
assert_eq!(server.requests.lock().unwrap().len(), 1);
|
||||
}
|
||||
for body in [
|
||||
@@ -222,7 +222,7 @@ async fn tmdb_all_routes_preserve_headers_with_no_query_token_or_details() {
|
||||
let mut selector = selector(100);
|
||||
selector.kind = kind;
|
||||
selector.time_window = window;
|
||||
assert!(client.fetch(&selector).await.unwrap().is_empty());
|
||||
assert_eq!(client.fetch(&selector).await.unwrap().len(), 0);
|
||||
let requests = server.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(
|
||||
@@ -285,7 +285,7 @@ async fn tmdb_identity_chunked_and_gzip_exact_and_plus_one_count_decoded_bytes()
|
||||
let mut budget = client.batch_budget();
|
||||
let result = client.trending(&selector(100), &mut budget).await;
|
||||
if length == BODY_LIMIT {
|
||||
assert!(result.unwrap().is_empty());
|
||||
assert_eq!(result.unwrap().len(), 0);
|
||||
} else {
|
||||
assert_eq!(result.unwrap_err(), TmdbFailure::BodyLimit);
|
||||
}
|
||||
|
||||
@@ -343,7 +343,7 @@ mod tests {
|
||||
assert_eq!(evaluation.selectors.len(), 1);
|
||||
assert_eq!(evaluation.selectors[0].reference_count, 0);
|
||||
assert_eq!(evaluation.selectors[0].membership_count, 0);
|
||||
assert!(evaluation.memberships.is_empty());
|
||||
assert_eq!(evaluation.memberships.len(), 0);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
|
||||
@@ -320,7 +320,7 @@ mod tests {
|
||||
None,
|
||||
);
|
||||
assert_eq!(preview.severity, ConflictSeverity::NoKnownConflict);
|
||||
assert!(preview.overlap_segments.is_empty());
|
||||
assert_eq!(preview.overlap_segments.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -706,7 +706,7 @@ mod tests {
|
||||
let dir = TempDir::new().expect("tempdir");
|
||||
let (queue, shared_file, target) = delete_one_of_two_entries_sharing(&dir).await;
|
||||
assert!(target.still_referenced, "Bob still holds this file");
|
||||
assert!(target.paths_to_unlink().is_empty());
|
||||
assert_eq!(target.paths_to_unlink().len(), 0);
|
||||
let unlinked = execute_deletion_target(&target).await.expect("execute");
|
||||
assert_eq!(unlinked, None, "nothing was unlinked");
|
||||
assert!(shared_file.exists(), "Bob's recording must survive Alice's deletion");
|
||||
|
||||
@@ -233,7 +233,7 @@ fn the_entry_behind_a_finished_transfer_attaches_instead_of_downloading_again()
|
||||
};
|
||||
let promoted = promote_from_queue(&mut candidate);
|
||||
assert!(promoted.is_none(), "nothing needs to run");
|
||||
assert!(candidate.active.is_empty());
|
||||
assert_eq!(candidate.active.len(), 0);
|
||||
assert!(candidate.queue.is_empty(), "bob left the queue");
|
||||
let bob = candidate.finished.iter().find(|task| task.uuid == "bob").expect("bob is filed");
|
||||
assert_eq!(bob.state, RecordingTaskState::Completed);
|
||||
|
||||
@@ -355,7 +355,7 @@ mod tests {
|
||||
let mut t = completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1);
|
||||
t.recording.completed_at = None;
|
||||
let out = compute_candidates(&[t], &config, 1_000_000_000);
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -593,6 +593,6 @@ mod tests {
|
||||
let config = RetentionConfig::default();
|
||||
let tasks = vec![completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1)];
|
||||
let out = compute_candidates(&tasks, &config, 0);
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,7 +442,7 @@ mod tests {
|
||||
let key = occurrence_key("r1", &source(), "ch-1", 1_000, "t:my show");
|
||||
let tasks = vec![task("r1", &key)];
|
||||
let out = plan_materializations(&rules, &programmes, &tasks, &TombstoneSet::default(), 0, Some(10_000));
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -450,7 +450,7 @@ mod tests {
|
||||
let rules = vec![new_episode_rule()];
|
||||
let programmes = vec![epg_programme("My Show", -1, 100)];
|
||||
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000));
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -458,7 +458,7 @@ mod tests {
|
||||
let rules = vec![new_episode_rule()];
|
||||
let programmes = vec![epg_programme("My Show", 5_000, 6_000)];
|
||||
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(2_000));
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -468,7 +468,7 @@ mod tests {
|
||||
p.previously_shown = true;
|
||||
let programmes = vec![p];
|
||||
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000));
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -478,7 +478,7 @@ mod tests {
|
||||
let rules = vec![r];
|
||||
let programmes = vec![epg_programme("My Show", 1_000, 2_000)];
|
||||
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000));
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -502,7 +502,7 @@ mod tests {
|
||||
// Set the EPG horizon to 1 hour — the next Sunday is
|
||||
// days away.
|
||||
let out = plan_materializations(&rules, &[], &[], &TombstoneSet::default(), now, Some(3_600));
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -514,6 +514,6 @@ mod tests {
|
||||
|
||||
let out = plan_materializations(&[rule], &[], &[], &TombstoneSet::default(), 0, None);
|
||||
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(out.len(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,7 +405,7 @@ pub async fn recovery_decision_for(final_path: &Path, partial: &Path) -> Recover
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[cfg(unix)]
|
||||
#[cfg(target_os = "linux")]
|
||||
use super::spawn_retrying_busy_executable;
|
||||
use super::{
|
||||
build_recording_args, classify_ffmpeg_failure, recording_partial_path,
|
||||
@@ -778,7 +778,8 @@ mod tests {
|
||||
let _ = fs::remove_dir_all(&recording.file_dir);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
// Linux rejects execution while a writer is open; macOS permits it.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn spawn_retries_while_executable_is_open_for_writing() {
|
||||
let script = fake_ffmpeg_script("busy", "#!/bin/sh\nexit 0\n");
|
||||
@@ -970,8 +971,7 @@ mod tests {
|
||||
};
|
||||
assert!(generated.status.success(), "{}", String::from_utf8_lossy(&generated.stderr));
|
||||
let bytes = tokio::fs::read(&input).await?;
|
||||
// This address is outside the usual environment's localhost exclusion.
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.2:0").await?;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
|
||||
let url =
|
||||
reqwest::Url::parse(&format!("http://{}/api/v1/playlist/recording/secret/live", listener.local_addr()?))?;
|
||||
let (request_tx, request_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -991,7 +991,12 @@ mod tests {
|
||||
Ok::<_, std::io::Error>(())
|
||||
});
|
||||
let wrapper = directory.path().join("ffmpeg-with-proxy");
|
||||
tokio::fs::write(&wrapper, "#!/bin/sh\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n").await?;
|
||||
// Explicitly remove proxy exclusions so localhost still exercises the bypass.
|
||||
tokio::fs::write(
|
||||
&wrapper,
|
||||
"#!/bin/sh\nunset no_proxy NO_PROXY\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n",
|
||||
)
|
||||
.await?;
|
||||
fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755))?;
|
||||
let mut recording = make_recording(chrono::Utc::now().timestamp(), 60);
|
||||
recording.url = url;
|
||||
|
||||
@@ -276,7 +276,7 @@ mod tests {
|
||||
let queue = RecordingQueue::new();
|
||||
let snapshot = recording_snapshot(&queue, &no_subject_claims(), &bare_app_config()).await;
|
||||
assert_eq!(snapshot.revision.0, 0);
|
||||
assert!(snapshot.tasks.is_empty());
|
||||
assert_eq!(snapshot.tasks.len(), 0);
|
||||
assert_eq!(snapshot.quota, RecordingQuotaSummaryDto::default(), "a session with no subject is charged nothing");
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ mod tests {
|
||||
let queue = RecordingQueue::new();
|
||||
let snapshot = recording_snapshot(&queue, &no_read_claims(), &bare_app_config()).await;
|
||||
assert_eq!(snapshot.revision.0, 0);
|
||||
assert!(snapshot.tasks.is_empty());
|
||||
assert_eq!(snapshot.tasks.len(), 0);
|
||||
}
|
||||
|
||||
/// Every internal fact a recording carries that a client must never
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
//! Publish degraded origin evidence before the manifest retry loop starts waiting.
|
||||
|
||||
use super::reevaluation::register_terminal_reserve_reevaluation;
|
||||
use crate::{
|
||||
manifest_fetch::{HlsOriginManifestFetchContext, OriginManifestFetchError},
|
||||
origin_progress::HlsOriginPathCondition,
|
||||
post_refresh_availability::commit_post_refresh_terminal_fallback,
|
||||
refresh::{HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) async fn record_retryable_manifest_attempt_failure(
|
||||
context: &HlsOriginManifestFetchContext,
|
||||
expected_progress_generation: u64,
|
||||
error: &OriginManifestFetchError,
|
||||
now_ms: u64,
|
||||
) {
|
||||
let action = {
|
||||
let mut session = context.session.write().await;
|
||||
if session.is_gc_marked_for_removal()
|
||||
|| session.origin_control.progress_generation != expected_progress_generation
|
||||
{
|
||||
return;
|
||||
}
|
||||
if matches!(error, OriginManifestFetchError::RetryableStatus(..)) {
|
||||
session.origin_control.record_origin_response(now_ms);
|
||||
}
|
||||
// An individual soft failure must not weaken an already established hard failure or conflict.
|
||||
if matches!(
|
||||
session.origin_control.path_condition,
|
||||
HlsOriginPathCondition::ProgressExpected | HlsOriginPathCondition::RetryableFetchFailure
|
||||
) {
|
||||
session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure;
|
||||
}
|
||||
HlsPostRefreshAvailabilityAction::Reevaluate {
|
||||
reason: HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
|
||||
origin_progress_generation: session.origin_control.progress_generation,
|
||||
media_readiness_generation: session.activity.media_readiness_generation,
|
||||
}
|
||||
};
|
||||
let Some(ctx) = context.availability_runtime.as_ref().and_then(crate::hls_ctx::WeakHlsCtx::upgrade) else {
|
||||
return;
|
||||
};
|
||||
let registration = register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(&context.session)).await;
|
||||
let _ = commit_post_refresh_terminal_fallback(ctx, Arc::clone(&context.session), action, registration).await;
|
||||
}
|
||||
@@ -17,11 +17,18 @@ use super::{
|
||||
HlsLiveReserveDeadline, HlsOriginProgressDecision, ProxySessionId,
|
||||
};
|
||||
|
||||
mod manifest_failure;
|
||||
mod recovery_pressure;
|
||||
mod reevaluation;
|
||||
mod terminal_cutover;
|
||||
mod terminal_pending_owner;
|
||||
pub(crate) use self::terminal_cutover::commit_terminal_tail_if_lease_reserve_requires_cutover_detailed;
|
||||
pub(crate) use self::{
|
||||
manifest_failure::record_retryable_manifest_attempt_failure,
|
||||
terminal_cutover::{
|
||||
commit_terminal_tail_if_lease_reserve_requires_cutover_detailed, terminal_bundle_failure_compatibility,
|
||||
terminal_bundle_incompatibility,
|
||||
},
|
||||
};
|
||||
use self::{
|
||||
recovery_pressure::{
|
||||
evaluate_and_commit_session_recovery_pressure_in_snapshot, hls_recovery_pressure_policy,
|
||||
|
||||
@@ -347,7 +347,7 @@ async fn evaluate_post_refresh_terminal_attempt(
|
||||
cycle: &mut HlsAvailabilityReevaluationCycle,
|
||||
now_ms: u64,
|
||||
) -> HlsTerminalReevaluationDecision {
|
||||
if !matches!(mode, HlsAvailabilityReevaluationMode::PostRefresh(_)) {
|
||||
if matches!(mode, HlsAvailabilityReevaluationMode::RecoveryPressure) {
|
||||
return HlsTerminalReevaluationDecision::Continue { terminal_evaluation_pending: false };
|
||||
}
|
||||
match evaluate_active_terminal_leases_for_reevaluation(ctx, session, &owner_key.proxy_session_id, now_ms).await {
|
||||
@@ -439,7 +439,7 @@ async fn run_hls_availability_reevaluation_attempt(
|
||||
session: &mut HlsSessionHandle,
|
||||
owner_key: &mut HlsAvailabilityReevaluationOwnerKey,
|
||||
ownership: &HlsAvailabilityReevaluationOwnership,
|
||||
refresh_request: &mut OriginRefreshRequest,
|
||||
refresh_request: &mut Option<OriginRefreshRequest>,
|
||||
cycle: &mut HlsAvailabilityReevaluationCycle,
|
||||
) -> HlsAvailabilityCycleDecision {
|
||||
if !ownership.is_current(owner_key) {
|
||||
@@ -463,6 +463,9 @@ async fn run_hls_availability_reevaluation_attempt(
|
||||
HlsTerminalReevaluationDecision::RestartCycle => return HlsAvailabilityCycleDecision::RestartCycle,
|
||||
HlsTerminalReevaluationDecision::Stop => return HlsAvailabilityCycleDecision::Stop,
|
||||
};
|
||||
let Some(refresh_request) = refresh_request.as_mut() else {
|
||||
return HlsAvailabilityCycleDecision::FinishCycle;
|
||||
};
|
||||
let schedule = match evaluate_availability_refresh_attempt(
|
||||
ctx,
|
||||
session,
|
||||
@@ -482,10 +485,13 @@ async fn run_hls_availability_reevaluation_attempt(
|
||||
return HlsAvailabilityCycleDecision::FinishCycle;
|
||||
}
|
||||
let retry_from_ms = ctx.hls_proxy.now_ms();
|
||||
let Some(retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else {
|
||||
let Some(mut retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else {
|
||||
warn!("HLS availability reevaluation cycle stopped: reason=deadline_elapsed");
|
||||
return HlsAvailabilityCycleDecision::FinishCycle;
|
||||
};
|
||||
if let Some(deadline) = cycle.live_reserve_deadline {
|
||||
retry_at_ms = retry_at_ms.min(deadline.next_reevaluation_at_ms);
|
||||
}
|
||||
match wait_for_availability_owner_signal(ownership, retry_at_ms, retry_from_ms).await {
|
||||
HlsAvailabilityOwnerWaitOutcome::Cancelled => {
|
||||
ownership.discard_superseded(owner_key);
|
||||
@@ -504,7 +510,7 @@ async fn run_hls_availability_reevaluation(
|
||||
mut session: HlsSessionHandle,
|
||||
mut owner_key: HlsAvailabilityReevaluationOwnerKey,
|
||||
ownership: HlsAvailabilityReevaluationOwnership,
|
||||
mut refresh_request: OriginRefreshRequest,
|
||||
mut refresh_request: Option<OriginRefreshRequest>,
|
||||
) {
|
||||
'cycles: loop {
|
||||
let mut cycle = HlsAvailabilityReevaluationCycle::new(ctx.hls_proxy.now_ms());
|
||||
@@ -605,6 +611,24 @@ pub(super) fn register_hls_availability_reevaluation_with_mode(
|
||||
let coordinator = ctx.hls_proxy.availability_reevaluations();
|
||||
let registration_key = owner_key.clone();
|
||||
coordinator.register(registration_key, mode, move |ownership| {
|
||||
run_hls_availability_reevaluation(ctx, session, owner_key, ownership, refresh_request)
|
||||
run_hls_availability_reevaluation(ctx, session, owner_key, ownership, Some(refresh_request))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn register_terminal_reserve_reevaluation(
|
||||
ctx: HlsCtx,
|
||||
session: HlsSessionHandle,
|
||||
) -> futures::future::BoxFuture<'static, HlsAvailabilityReevaluationRegistration> {
|
||||
Box::pin(async move {
|
||||
let proxy_session_id = session.read().await.proxy_session_id.clone();
|
||||
let Some(owner_key) = ctx.hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await
|
||||
else {
|
||||
return HlsAvailabilityReevaluationRegistration::Superseded;
|
||||
};
|
||||
ctx.hls_proxy.availability_reevaluations().register(
|
||||
owner_key.clone(),
|
||||
HlsAvailabilityReevaluationMode::TerminalReserve,
|
||||
move |ownership| run_hls_availability_reevaluation(ctx, session, owner_key, ownership, None),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,9 +8,9 @@
|
||||
use super::{
|
||||
recovery_pressure::{hls_recovery_pressure_policy, recovery_trigger_budget},
|
||||
terminal_pending_owner::{
|
||||
commit_prepared_terminal_unavailable_after_owner_failure, commit_terminal_unavailable,
|
||||
register_ready_terminal_pending_owner, register_terminal_pending_owner, terminal_asset_revision_guard,
|
||||
terminal_resolution_with_failed_closed_fallback,
|
||||
commit_late_custom_response, commit_prepared_terminal_unavailable_after_owner_failure,
|
||||
commit_terminal_unavailable, register_ready_terminal_pending_owner, register_terminal_pending_owner,
|
||||
terminal_asset_revision_guard, terminal_resolution_with_failed_closed_fallback,
|
||||
},
|
||||
HlsDetailedTerminalResolution, HlsTerminalDecisionPurpose, HlsTerminalFailedClosedReason, HlsTerminalResolution,
|
||||
HLS_PLAYBACK_RATE_GUARD_MILLI,
|
||||
@@ -184,6 +184,7 @@ async fn resolve_terminal_cutover_before_commit_window(
|
||||
&context.lease.lease_id,
|
||||
&preparation,
|
||||
context.now_ms,
|
||||
HlsTerminalTailCompatibility::TerminalMediaNotReady,
|
||||
),
|
||||
context.evaluation.safe_deadline,
|
||||
);
|
||||
@@ -239,7 +240,7 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
lease: &HlsAccessLease,
|
||||
now_ms: u64,
|
||||
) -> HlsTerminalResolution {
|
||||
commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
|
||||
let detailed = commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
|
||||
ctx,
|
||||
session,
|
||||
proxy_session_id,
|
||||
@@ -247,8 +248,30 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
now_ms,
|
||||
HlsTerminalDecisionPurpose::OrdinaryCutover,
|
||||
)
|
||||
.await
|
||||
.resolution
|
||||
.await;
|
||||
if detailed.live_reserve_deadline.is_some()
|
||||
&& matches!(detailed.resolution, HlsTerminalResolution::LiveAllowed | HlsTerminalResolution::Reevaluate)
|
||||
{
|
||||
let registration =
|
||||
super::reevaluation::register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(session)).await;
|
||||
if matches!(
|
||||
registration,
|
||||
crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::CapacityExceeded
|
||||
| crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::RuntimeUnavailable
|
||||
) {
|
||||
return commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
|
||||
ctx,
|
||||
session,
|
||||
proxy_session_id,
|
||||
lease,
|
||||
now_ms,
|
||||
HlsTerminalDecisionPurpose::AutonomousOwnerFailureFallback,
|
||||
)
|
||||
.await
|
||||
.resolution;
|
||||
}
|
||||
}
|
||||
detailed.resolution
|
||||
}
|
||||
|
||||
pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
|
||||
@@ -341,13 +364,26 @@ async fn commit_prepared_terminal_decision(
|
||||
preparation: &HlsTerminalTailPreparation,
|
||||
now_ms: u64,
|
||||
) -> HlsTerminalResolution {
|
||||
if !preparation.trigger.is_runtime_policy()
|
||||
&& now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
|
||||
{
|
||||
return commit_late_custom_response(HlsTerminalCommitContext {
|
||||
ctx,
|
||||
session,
|
||||
proxy_session_id,
|
||||
lease_id,
|
||||
preparation,
|
||||
now_ms,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
let reason = preparation.trigger.reason();
|
||||
let asset = match snapshot_hls_runtime_custom_tail_asset(ctx, reason) {
|
||||
Ok(asset) => asset,
|
||||
Err(compatibility) => {
|
||||
let context = HlsTerminalCommitContext { ctx, session, proxy_session_id, lease_id, preparation, now_ms };
|
||||
let outcome = commit_terminal_unavailable(context, None, compatibility.terminal_compatibility());
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
|
||||
}
|
||||
};
|
||||
commit_prepared_runtime_custom_tail(ctx, session, proxy_session_id, lease_id, preparation, now_ms, asset).await
|
||||
@@ -366,6 +402,11 @@ pub async fn commit_prepared_runtime_custom_tail(
|
||||
if preparation.trigger.reason() != asset.reason {
|
||||
return HlsTerminalResolution::Reevaluate;
|
||||
}
|
||||
if !preparation.trigger.is_runtime_policy()
|
||||
&& now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
|
||||
{
|
||||
return commit_late_custom_response(context).await;
|
||||
}
|
||||
let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&asset);
|
||||
let media_asset = asset.asset;
|
||||
let static_incompatibility = if preparation.manifest_snapshot.delivery_mode
|
||||
@@ -381,7 +422,7 @@ pub async fn commit_prepared_runtime_custom_tail(
|
||||
};
|
||||
if let Some(reason) = static_incompatibility {
|
||||
let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
|
||||
}
|
||||
let bundle_key = prepared_terminal_bundle_key(
|
||||
&media_asset,
|
||||
@@ -403,7 +444,7 @@ pub async fn commit_prepared_runtime_custom_tail(
|
||||
Some(expected_asset),
|
||||
HlsTerminalTailCompatibility::AssetRevisionMismatch,
|
||||
);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
|
||||
}
|
||||
HlsPreparedTerminalBundleState::Preparing { .. } => {
|
||||
return register_terminal_pending_owner(context, media_asset, expected_asset, bundle_key).await;
|
||||
@@ -411,12 +452,12 @@ pub async fn commit_prepared_runtime_custom_tail(
|
||||
HlsPreparedTerminalBundleState::Failed { key, reason } => {
|
||||
let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason);
|
||||
let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
|
||||
}
|
||||
HlsPreparedTerminalBundleState::Incompatible { reason, .. } => {
|
||||
let reason = terminal_bundle_incompatibility(reason);
|
||||
let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome);
|
||||
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
|
||||
}
|
||||
};
|
||||
if preparation.trigger.is_runtime_policy() {
|
||||
@@ -426,10 +467,11 @@ pub async fn commit_prepared_runtime_custom_tail(
|
||||
expected_asset,
|
||||
bundle_key,
|
||||
prepared_bundle,
|
||||
);
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let outcome = commit_ready_terminal_bundle(context, media_asset, expected_asset, bundle_key, prepared_bundle).await;
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome)
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome).await
|
||||
}
|
||||
|
||||
struct HlsReadyTerminalSplice {
|
||||
@@ -624,7 +666,7 @@ pub(super) async fn commit_ready_terminal_bundle(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn terminal_bundle_incompatibility(
|
||||
pub(crate) fn terminal_bundle_incompatibility(
|
||||
reason: HlsPreparedTerminalBundleIncompatibility,
|
||||
) -> HlsTerminalTailCompatibility {
|
||||
match reason {
|
||||
@@ -636,7 +678,7 @@ pub(super) fn terminal_bundle_incompatibility(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn terminal_bundle_failure_compatibility(
|
||||
pub(crate) fn terminal_bundle_failure_compatibility(
|
||||
actual_key: HlsPreparedTerminalBundleKey,
|
||||
required_key: HlsPreparedTerminalBundleKey,
|
||||
reason: HlsPreparedTerminalBundleFailure,
|
||||
|
||||
@@ -89,7 +89,7 @@ pub(super) async fn register_terminal_pending_owner(
|
||||
ownership,
|
||||
)
|
||||
});
|
||||
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration)
|
||||
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await
|
||||
}
|
||||
|
||||
fn terminal_pending_owner_key(
|
||||
@@ -117,7 +117,7 @@ fn terminal_pending_owner_key(
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn register_ready_terminal_pending_owner(
|
||||
pub(super) async fn register_ready_terminal_pending_owner(
|
||||
context: HlsTerminalCommitContext<'_>,
|
||||
asset: Arc<crate::terminal_tail::HlsTerminalMediaAsset>,
|
||||
expected_asset: HlsRuntimeCustomTailAssetIdentity,
|
||||
@@ -150,16 +150,15 @@ pub(super) fn register_ready_terminal_pending_owner(
|
||||
};
|
||||
let outcome = commit_ready_terminal_bundle(task_context, asset, expected_asset, bundle_key, bundle).await;
|
||||
if ownership.is_current() {
|
||||
observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(
|
||||
task_context,
|
||||
outcome,
|
||||
));
|
||||
observe_autonomous_terminal_resolution(
|
||||
terminal_resolution_with_failed_closed_fallback(task_context, outcome).await,
|
||||
);
|
||||
}
|
||||
});
|
||||
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration)
|
||||
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await
|
||||
}
|
||||
|
||||
pub(super) fn terminal_resolution_for_pending_registration(
|
||||
pub(super) async fn terminal_resolution_for_pending_registration(
|
||||
context: HlsTerminalCommitContext<'_>,
|
||||
expected_asset: HlsRuntimeCustomTailAssetIdentity,
|
||||
latest_safe_commit_at_ms: u64,
|
||||
@@ -178,7 +177,7 @@ pub(super) fn terminal_resolution_for_pending_registration(
|
||||
Some(expected_asset),
|
||||
HlsTerminalTailCompatibility::TerminalMediaNotReady,
|
||||
);
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome)
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome).await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -193,7 +192,7 @@ async fn terminal_resolution_after_settled_bundle_observation(
|
||||
match state {
|
||||
Some(HlsPreparedTerminalBundleState::Ready { bundle }) if bundle.matches_key_and_shape(bundle_key) => {
|
||||
let outcome = commit_ready_terminal_bundle(context, asset, expected_asset, bundle_key, bundle).await;
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome)
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome).await
|
||||
}
|
||||
Some(HlsPreparedTerminalBundleState::Ready { .. }) => {
|
||||
let outcome = commit_terminal_unavailable(
|
||||
@@ -201,12 +200,12 @@ async fn terminal_resolution_after_settled_bundle_observation(
|
||||
Some(expected_asset),
|
||||
HlsTerminalTailCompatibility::AssetRevisionMismatch,
|
||||
);
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome)
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome).await
|
||||
}
|
||||
Some(HlsPreparedTerminalBundleState::Failed { key, reason }) => {
|
||||
let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason);
|
||||
let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility);
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome)
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome).await
|
||||
}
|
||||
Some(HlsPreparedTerminalBundleState::Incompatible { key, reason }) => {
|
||||
let compatibility = if key == bundle_key {
|
||||
@@ -215,7 +214,7 @@ async fn terminal_resolution_after_settled_bundle_observation(
|
||||
HlsTerminalTailCompatibility::AssetRevisionMismatch
|
||||
};
|
||||
let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility);
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome)
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome).await
|
||||
}
|
||||
Some(HlsPreparedTerminalBundleState::Preparing { .. }) | None => {
|
||||
let outcome = commit_terminal_unavailable(
|
||||
@@ -223,7 +222,7 @@ async fn terminal_resolution_after_settled_bundle_observation(
|
||||
Some(expected_asset),
|
||||
HlsTerminalTailCompatibility::TerminalMediaNotReady,
|
||||
);
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome)
|
||||
terminal_resolution_with_failed_closed_fallback(context, outcome).await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -330,7 +329,7 @@ pub(super) async fn run_terminal_pending_owner(
|
||||
commit_terminal_unavailable(context, Some(expected_asset), compatibility)
|
||||
}
|
||||
};
|
||||
observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome));
|
||||
observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome).await);
|
||||
}
|
||||
|
||||
fn terminal_pending_retry_after_ms(now_ms: u64, latest_safe_commit_at_ms: u64) -> u64 {
|
||||
@@ -378,7 +377,7 @@ pub(super) fn terminal_resolution_for_commit_outcome(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn terminal_resolution_with_failed_closed_fallback(
|
||||
pub(super) async fn terminal_resolution_with_failed_closed_fallback(
|
||||
context: HlsTerminalCommitContext<'_>,
|
||||
outcome: HlsTerminalCommitOutcome,
|
||||
) -> HlsTerminalResolution {
|
||||
@@ -389,6 +388,9 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback(
|
||||
if context.preparation.trigger.is_runtime_policy() {
|
||||
return resolution;
|
||||
}
|
||||
if outcome == HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed {
|
||||
return commit_late_custom_response(context).await;
|
||||
}
|
||||
commit_prepared_terminal_unavailable_after_owner_failure(
|
||||
context.ctx,
|
||||
context.session,
|
||||
@@ -396,9 +398,48 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback(
|
||||
context.lease_id,
|
||||
context.preparation,
|
||||
context.now_ms,
|
||||
HlsTerminalTailCompatibility::TerminalMediaNotReady,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn commit_late_custom_response(context: HlsTerminalCommitContext<'_>) -> HlsTerminalResolution {
|
||||
let prepared = crate::late_custom_response::prepare_late_custom_response(
|
||||
context.ctx,
|
||||
context.proxy_session_id,
|
||||
context.lease_id,
|
||||
context.preparation,
|
||||
context.now_ms,
|
||||
)
|
||||
.await;
|
||||
let (plan, expected_asset) = match prepared {
|
||||
Ok(prepared) => prepared,
|
||||
Err(crate::late_custom_response::HlsLateCustomResponseError::Superseded) => {
|
||||
return HlsTerminalResolution::Reevaluate
|
||||
}
|
||||
Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) => {
|
||||
return commit_prepared_terminal_unavailable_after_owner_failure(
|
||||
context.ctx,
|
||||
context.session,
|
||||
context.proxy_session_id,
|
||||
context.lease_id,
|
||||
context.preparation,
|
||||
context.now_ms,
|
||||
reason,
|
||||
)
|
||||
}
|
||||
};
|
||||
let outcome = context.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest {
|
||||
session: context.session,
|
||||
lease_id: context.lease_id,
|
||||
proxy_session_id: context.proxy_session_id,
|
||||
preparation: context.preparation,
|
||||
now_ms: context.ctx.hls_proxy.terminal_commit_now_ms().max(context.now_ms),
|
||||
payload: HlsTerminalCommitPayload::CustomResponse(plan),
|
||||
asset_revision_guard: terminal_asset_revision_guard(context.ctx, expected_asset.reason, Some(expected_asset)),
|
||||
});
|
||||
terminal_resolution_for_commit_outcome(outcome, context.now_ms)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum HlsAutonomousTerminalObservation {
|
||||
Committed,
|
||||
@@ -486,6 +527,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure(
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &HlsTerminalTailPreparation,
|
||||
now_ms: u64,
|
||||
reason: HlsTerminalTailCompatibility,
|
||||
) -> HlsTerminalResolution {
|
||||
let expected_asset = configured_terminal_asset_identity(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable);
|
||||
let outcome = ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest {
|
||||
@@ -494,9 +536,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure(
|
||||
proxy_session_id,
|
||||
preparation,
|
||||
now_ms: ctx.hls_proxy.terminal_commit_now_ms().max(now_ms),
|
||||
payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure(
|
||||
HlsTerminalTailCompatibility::TerminalMediaNotReady,
|
||||
),
|
||||
payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure(reason),
|
||||
asset_revision_guard: terminal_asset_revision_guard(
|
||||
ctx,
|
||||
HlsRuntimeCustomTailReason::ChannelUnavailable,
|
||||
|
||||
@@ -212,6 +212,7 @@ mod policy;
|
||||
|
||||
mod publication;
|
||||
|
||||
mod late_custom_response;
|
||||
mod recovery;
|
||||
|
||||
mod storage;
|
||||
|
||||
@@ -0,0 +1,532 @@
|
||||
use super::*;
|
||||
use crate::{
|
||||
late_custom_response::prepare_late_custom_response,
|
||||
manager::{HlsTerminalCommitPayload, HlsTerminalCommitRequest},
|
||||
};
|
||||
|
||||
async fn late_fixture(name: &str) -> HlsTerminalPendingCommitFixture {
|
||||
let fixture = terminal_pending_commit_fixture(name).await;
|
||||
fixture.session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict;
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn prepare(
|
||||
fixture: &HlsTerminalPendingCommitFixture,
|
||||
) -> (Arc<crate::late_custom_response::HlsLateCustomResponsePlan>, HlsRuntimeCustomTailAssetIdentity) {
|
||||
prepare_late_custom_response(
|
||||
&fixture.ctx,
|
||||
&fixture.proxy_session_id,
|
||||
&fixture.lease_id,
|
||||
&fixture.preparation,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("late custom plan"))
|
||||
}
|
||||
|
||||
fn commit(
|
||||
fixture: &HlsTerminalPendingCommitFixture,
|
||||
plan: Arc<crate::late_custom_response::HlsLateCustomResponsePlan>,
|
||||
expected_asset: HlsRuntimeCustomTailAssetIdentity,
|
||||
now_ms: u64,
|
||||
) -> HlsTerminalCommitOutcome {
|
||||
fixture.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest {
|
||||
session: &fixture.session,
|
||||
lease_id: &fixture.lease_id,
|
||||
proxy_session_id: &fixture.proxy_session_id,
|
||||
preparation: &fixture.preparation,
|
||||
now_ms,
|
||||
payload: HlsTerminalCommitPayload::CustomResponse(plan),
|
||||
asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, expected_asset.reason, Some(expected_asset)),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_commits_without_cached_base_and_serves_frozen_media() {
|
||||
let fixture = late_fixture("late-without-base").await;
|
||||
let (plan, asset) = prepare(&fixture).await;
|
||||
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(asset.media);
|
||||
assert!(
|
||||
fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms)
|
||||
.await
|
||||
.is_err(),
|
||||
"preparation cannot authorize uncommitted custom media"
|
||||
);
|
||||
fixture.session.write().await.segments.clear();
|
||||
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::Committed);
|
||||
assert!(!fixture.session.read().await.has_terminal_tail_protections());
|
||||
assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:41\n"));
|
||||
assert!(!plan.manifest_body().contains("/hls/shared/live/"));
|
||||
let segment = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms)
|
||||
.await
|
||||
.expect("committed late media");
|
||||
assert!(!segment.bytes.is_empty());
|
||||
fixture.ctx.app_config.custom_stream_response.store(None);
|
||||
assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms + 1), HlsTerminalCommitOutcome::AlreadyCommitted);
|
||||
let replay = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms + 1)
|
||||
.await
|
||||
.expect("frozen media after config reload");
|
||||
assert_eq!(replay.bytes, segment.bytes);
|
||||
assert!(fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, "wrong", 11, now_ms)
|
||||
.await
|
||||
.is_err());
|
||||
assert!(fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 12, now_ms)
|
||||
.await
|
||||
.is_err());
|
||||
assert!(fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms + 60_000)
|
||||
.await
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_does_not_commit_over_recovered_origin_or_changed_asset() {
|
||||
for changed_asset in [false, true] {
|
||||
let fixture = late_fixture(if changed_asset { "late-asset-changed" } else { "late-origin-recovered" }).await;
|
||||
let (plan, asset) = prepare(&fixture).await;
|
||||
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
let expected = if changed_asset {
|
||||
fixture.ctx.app_config.custom_stream_response.store(None);
|
||||
HlsTerminalCommitOutcome::SupersededGeneration
|
||||
} else {
|
||||
fixture.session.write().await.origin_control.record_media_progress(now_ms, 12_000);
|
||||
HlsTerminalCommitOutcome::RecoveryCommitted
|
||||
};
|
||||
assert_eq!(commit(&fixture, plan, asset, now_ms), expected);
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
|
||||
.await
|
||||
.expect("lease");
|
||||
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_preserves_the_prepared_asset_reason_guard() {
|
||||
let fixture = late_fixture("late-wrong-asset-reason").await;
|
||||
let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted)
|
||||
.expect("configured alternate custom asset");
|
||||
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
assert_eq!(
|
||||
commit_prepared_runtime_custom_tail(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&fixture.lease_id,
|
||||
&fixture.preparation,
|
||||
now_ms,
|
||||
asset,
|
||||
)
|
||||
.await,
|
||||
HlsTerminalResolution::Reevaluate
|
||||
);
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
|
||||
.await
|
||||
.expect("live lease");
|
||||
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_preserves_target_duration_exceeded_through_terminal_commit() {
|
||||
let mut fixture = late_fixture("late-target-duration-error").await;
|
||||
let mut manifest = fixture.preparation.manifest_snapshot.clone();
|
||||
manifest.target_duration_ms = 1_000;
|
||||
manifest.playlist_duration_ms = 1_000;
|
||||
manifest.last_visible_media_end_ms = 1_000;
|
||||
Arc::make_mut(&mut manifest.visible_segments)[0].duration_ms = 1_000;
|
||||
let guard = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("publication guard");
|
||||
assert!(fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.commit_access_lease_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
manifest,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await
|
||||
.is_committed());
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("lease with short target duration");
|
||||
fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("published snapshot");
|
||||
fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation;
|
||||
let expected = HlsTerminalTailCompatibility::TargetDurationExceeded {
|
||||
asset_ms: fixture.asset.duration_ms(),
|
||||
target_ms: 1_000,
|
||||
};
|
||||
assert!(matches!(prepare_late_custom_response(
|
||||
&fixture.ctx, &fixture.proxy_session_id, &fixture.lease_id, &fixture.preparation, fixture.now_ms,
|
||||
).await, Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) if reason == expected));
|
||||
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
assert_eq!(
|
||||
super::super::terminal_pending_owner::commit_late_custom_response(HlsTerminalCommitContext {
|
||||
ctx: &fixture.ctx,
|
||||
session: &fixture.session,
|
||||
proxy_session_id: &fixture.proxy_session_id,
|
||||
lease_id: &fixture.lease_id,
|
||||
preparation: &fixture.preparation,
|
||||
now_ms,
|
||||
})
|
||||
.await,
|
||||
HlsTerminalResolution::Committed
|
||||
);
|
||||
let terminal = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
|
||||
.await
|
||||
.expect("terminal unavailable lease");
|
||||
assert_eq!(
|
||||
terminal.playback_mode,
|
||||
HlsLeasePlaybackMode::TerminalUnavailable {
|
||||
decision_generation: fixture.preparation.decision_generation,
|
||||
reason: expected,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async fn committed_late_activity_fixture(name: &str) -> HlsTerminalPendingCommitFixture {
|
||||
let fixture = late_fixture(name).await;
|
||||
let mut config = fixture.ctx.app_config.config.load_full().as_ref().clone();
|
||||
config.reverse_proxy =
|
||||
Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto {
|
||||
hls_cache: Some(shared::model::HlsCacheConfigDto {
|
||||
session_idle_timeout: shared::model::Secs::new(60),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}));
|
||||
fixture.ctx.app_config.config.store(Arc::new(config));
|
||||
fixture.ctx.hls_proxy.update_config(&fixture.ctx.app_config).await;
|
||||
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
assert!(fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.activate_access_lease(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
now_ms,
|
||||
HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 },
|
||||
)
|
||||
.await
|
||||
.is_activated());
|
||||
let (plan, asset) = prepare(&fixture).await;
|
||||
assert_eq!(commit(&fixture, plan, asset, now_ms), HlsTerminalCommitOutcome::Committed);
|
||||
fixture
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_media_requests_keep_lease_and_session_alive_beyond_initial_sixty_seconds() {
|
||||
use crate::{
|
||||
lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey},
|
||||
HlsMediaActivityCommitOutcome,
|
||||
};
|
||||
let fixture = committed_late_activity_fixture("late-active-media").await;
|
||||
let manager = &fixture.ctx.hls_proxy;
|
||||
let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
let before = manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms)
|
||||
.await
|
||||
.expect("terminal lease");
|
||||
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media);
|
||||
for index in 1_u16..=11 {
|
||||
let now_ms = started_at_ms + u64::from(index) * 10_000;
|
||||
let access = manager
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, index, now_ms)
|
||||
.await
|
||||
.expect("custom media remains available");
|
||||
assert_eq!(
|
||||
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, now_ms).await,
|
||||
HlsMediaActivityCommitOutcome::Committed
|
||||
);
|
||||
if index == 6 {
|
||||
for key in [
|
||||
HlsLifecycleEventKey::AccessLeaseValidity {
|
||||
lease_id: fixture.lease_id.clone(),
|
||||
proxy_session_id: fixture.proxy_session_id.clone(),
|
||||
},
|
||||
HlsLifecycleEventKey::SessionIdle { proxy_session_id: fixture.proxy_session_id.clone() },
|
||||
] {
|
||||
manager
|
||||
.handle_lifecycle_event(
|
||||
&fixture.ctx.active_users,
|
||||
&fixture.ctx.active_provider,
|
||||
HlsLifecycleEvent { key, due_at_ms: started_at_ms + 60_000 },
|
||||
now_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
let lease = manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
|
||||
.await
|
||||
.expect("active terminal lease");
|
||||
assert_eq!(lease.state, HlsAccessLeaseState::Activated);
|
||||
assert_eq!(lease.last_seen_at_ms, now_ms);
|
||||
assert_eq!(lease.valid_until_ms, now_ms + 60_000);
|
||||
assert!(lease.active_until_ms.is_some_and(|until| until > now_ms));
|
||||
assert_eq!(lease.playback_mode, before.playback_mode);
|
||||
assert_eq!(lease.playback_cursor, before.playback_cursor);
|
||||
assert_eq!(lease.admission_generation, before.admission_generation);
|
||||
assert_eq!(fixture.session.read().await.last_client_access_at_ms, now_ms);
|
||||
}
|
||||
let expired_at_ms = started_at_ms + 170_000;
|
||||
assert!(manager
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, expired_at_ms)
|
||||
.await
|
||||
.is_err());
|
||||
let expired = manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expired_at_ms)
|
||||
.await
|
||||
.expect("expired lease");
|
||||
assert_eq!(expired.state, HlsAccessLeaseState::Expired);
|
||||
assert_eq!(expired.last_seen_at_ms, started_at_ms + 110_000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_media_activity_reactivates_idle_lease_but_rejects_stale_or_expired_access() {
|
||||
use crate::{
|
||||
lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey},
|
||||
HlsMediaActivityCommitOutcome,
|
||||
};
|
||||
let fixture = committed_late_activity_fixture("late-idle-media").await;
|
||||
let manager = &fixture.ctx.hls_proxy;
|
||||
let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
manager
|
||||
.handle_lifecycle_event(
|
||||
&fixture.ctx.active_users,
|
||||
&fixture.ctx.active_provider,
|
||||
HlsLifecycleEvent {
|
||||
key: HlsLifecycleEventKey::AccessLeaseActive {
|
||||
lease_id: fixture.lease_id.clone(),
|
||||
proxy_session_id: fixture.proxy_session_id.clone(),
|
||||
},
|
||||
due_at_ms: started_at_ms + 30_000,
|
||||
},
|
||||
started_at_ms + 40_000,
|
||||
)
|
||||
.await;
|
||||
let idle = manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
|
||||
.await
|
||||
.expect("idle lease");
|
||||
assert_eq!(idle.state, HlsAccessLeaseState::Idle);
|
||||
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media);
|
||||
let mut access = manager
|
||||
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, started_at_ms + 40_000)
|
||||
.await
|
||||
.expect("idle custom media");
|
||||
let original_generation = access.terminal_generation;
|
||||
access.terminal_generation = Some(crate::HlsTerminalTailGeneration(fixture.preparation.decision_generation + 1));
|
||||
assert_eq!(
|
||||
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await,
|
||||
HlsMediaActivityCommitOutcome::StaleLeaseIdentity
|
||||
);
|
||||
let unchanged = manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
|
||||
.await
|
||||
.expect("unchanged idle lease");
|
||||
assert_eq!(unchanged.last_seen_at_ms, idle.last_seen_at_ms);
|
||||
assert_eq!(unchanged.valid_until_ms, idle.valid_until_ms);
|
||||
access.terminal_generation = original_generation;
|
||||
assert_eq!(
|
||||
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await,
|
||||
HlsMediaActivityCommitOutcome::Committed
|
||||
);
|
||||
let active = manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
|
||||
.await
|
||||
.expect("reactivated lease");
|
||||
assert_eq!(active.state, HlsAccessLeaseState::Activated);
|
||||
assert_eq!(active.valid_until_ms, started_at_ms + 100_000);
|
||||
assert_eq!(
|
||||
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 100_000).await,
|
||||
HlsMediaActivityCommitOutcome::StaleLeaseIdentity
|
||||
);
|
||||
let expired = manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 100_000)
|
||||
.await
|
||||
.expect("expired lease");
|
||||
assert_eq!(expired.state, HlsAccessLeaseState::Expired);
|
||||
assert_eq!(expired.last_seen_at_ms, active.last_seen_at_ms);
|
||||
assert_eq!(expired.valid_until_ms, active.valid_until_ms);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_retries_contended_commit_once_and_remains_immutable() {
|
||||
let mut fixture = late_fixture("late-contended-commit").await;
|
||||
let now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms();
|
||||
fixture.preparation.cutover_timing =
|
||||
HlsLeaseCutoverTiming::from_reserve(now_ms, 0, fixture.preparation.reserve.transition_margin, None);
|
||||
let (plan, asset) = prepare(&fixture).await;
|
||||
let guard = fixture.session.read().await;
|
||||
assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. }));
|
||||
assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. }));
|
||||
drop(guard);
|
||||
let committed = tokio::time::timeout(Duration::from_secs(3), async {
|
||||
loop {
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
fixture.ctx.hls_proxy.terminal_commit_now_ms(),
|
||||
)
|
||||
.await
|
||||
.expect("lease");
|
||||
if let HlsLeasePlaybackMode::TerminalCustomResponse(committed) = lease.playback_mode {
|
||||
break committed;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("autonomous late custom commit");
|
||||
assert_eq!(committed, plan);
|
||||
assert_eq!(
|
||||
commit(&fixture, plan, asset, fixture.ctx.hls_proxy.terminal_commit_now_ms()),
|
||||
HlsTerminalCommitOutcome::AlreadyCommitted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_uses_highwater_even_when_latest_manifest_is_shorter() {
|
||||
let mut fixture = late_fixture("late-highwater").await;
|
||||
for (last_seq, discontinuity_sequence, discontinuity_before) in [(100, 7, true), (50, 3, false)] {
|
||||
let guard = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("publication guard");
|
||||
let mut manifest = fixture.preparation.manifest_snapshot.clone();
|
||||
manifest.first_proxy_seq = last_seq;
|
||||
manifest.last_proxy_seq = last_seq;
|
||||
manifest.discontinuity_sequence = discontinuity_sequence;
|
||||
let segment = Arc::make_mut(&mut manifest.visible_segments).first_mut().expect("segment");
|
||||
segment.proxy_seq = last_seq;
|
||||
segment.discontinuity_before = discontinuity_before;
|
||||
assert!(fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.commit_access_lease_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
manifest,
|
||||
fixture.now_ms
|
||||
)
|
||||
.await
|
||||
.is_committed());
|
||||
}
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("lease");
|
||||
fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("latest snapshot");
|
||||
fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation;
|
||||
let (plan, _) = prepare(&fixture).await;
|
||||
assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:101\n"));
|
||||
assert!(plan.manifest_body().contains("#EXT-X-DISCONTINUITY-SEQUENCE:8\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_custom_response_cas_budget_cannot_be_renewed_or_poisoned_by_polls() {
|
||||
use crate::terminal_commit::{
|
||||
terminal_commit_retry_schedule_budget_ms, HlsTerminalCommitCommand, HlsTerminalCommitOwnerKey,
|
||||
HlsTerminalCommitRetryCoordinator, HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision,
|
||||
};
|
||||
let fixture = late_fixture("late-bounded-cas").await;
|
||||
let (plan, asset) = prepare(&fixture).await;
|
||||
let coordinator = HlsTerminalCommitRetryCoordinator::default();
|
||||
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
|
||||
let command = |decision| {
|
||||
let (cancellation_epoch, submission_token) = coordinator.reserve_submission().expect("submission binding");
|
||||
HlsTerminalCommitCommand {
|
||||
key: HlsTerminalCommitOwnerKey {
|
||||
proxy_session_id: fixture.proxy_session_id.clone(),
|
||||
lease_id: fixture.lease_id.clone(),
|
||||
lease_issued_at_ms: fixture.preparation.lease_issued_at_ms,
|
||||
decision_generation: fixture.preparation.decision_generation,
|
||||
},
|
||||
session: Arc::clone(&fixture.session),
|
||||
session_incarnation: fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.sessions()
|
||||
.session_incarnation(&fixture.session)
|
||||
.expect("session incarnation"),
|
||||
preparation: fixture.preparation.clone(),
|
||||
decision,
|
||||
media_guard: None,
|
||||
asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, asset.reason, Some(asset)),
|
||||
cancellation_epoch,
|
||||
submission_token,
|
||||
}
|
||||
};
|
||||
assert!(matches!(
|
||||
coordinator.submit(
|
||||
command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)),
|
||||
now_ms
|
||||
),
|
||||
HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed)
|
||||
));
|
||||
assert!(matches!(
|
||||
coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms),
|
||||
HlsTerminalCommitSubmissionDecision::Attempt { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
coordinator.submit(
|
||||
command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)),
|
||||
now_ms + 1
|
||||
),
|
||||
HlsTerminalCommitSubmissionDecision::PendingExisting { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms + 100),
|
||||
HlsTerminalCommitSubmissionDecision::PendingExisting { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
coordinator.submit(
|
||||
command(HlsTerminalLeaseDecision::CustomResponse(plan)),
|
||||
now_ms + terminal_commit_retry_schedule_budget_ms()
|
||||
),
|
||||
HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed)
|
||||
));
|
||||
}
|
||||
@@ -31,6 +31,7 @@ pub(super) fn post_refresh_owner_request(fixture: &PostRefreshTerminalFixture) -
|
||||
manifest_commit_requirement:
|
||||
super::super::super::refresh::HlsManifestCommitRequirement::CommittedManifestAllowed,
|
||||
fresh_manifest_requirement_generation: None,
|
||||
refresh_identity: None,
|
||||
acceptance_directive: HlsManifestAcceptanceDirective::none(),
|
||||
access_lease_id: None,
|
||||
now_ms: fixture.now_ms,
|
||||
|
||||
@@ -386,3 +386,287 @@ fn hls_session_recovery_pressure_new_publication_lateness_uses_current_reserve_e
|
||||
assert!(pressure.decision.close_admission);
|
||||
assert!(!pressure.decision.evaluate_lease_cutovers);
|
||||
}
|
||||
|
||||
async fn wait_for_terminal_mode(fixture: &PostRefreshTerminalFixture) -> HlsLeasePlaybackMode {
|
||||
tokio::time::timeout(Duration::from_millis(4_500), async {
|
||||
loop {
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("active lease");
|
||||
if lease.playback_mode != HlsLeasePlaybackMode::Live {
|
||||
return lease.playback_mode;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("terminal decision before reserve is exhausted")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh_is_in_flight() {
|
||||
let fixture = post_refresh_terminal_fixture_with_progress("autonomous-reserve-deadline", true, false).await;
|
||||
advance_post_refresh_fixture_playback(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&fixture.lease_id,
|
||||
18_000,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await;
|
||||
{
|
||||
let mut session = fixture.session.write().await;
|
||||
session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure;
|
||||
session.origin_refresh.mark_started(fixture.now_ms);
|
||||
}
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("live lease");
|
||||
let result = commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&lease,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result, HlsTerminalResolution::LiveAllowed);
|
||||
assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1);
|
||||
|
||||
// R=17,100 ms and M=12,000 ms: the next six-second client poll would be too late.
|
||||
// Only read the resulting state; do not issue another terminal/manifest request.
|
||||
let mode = wait_for_terminal_mode(&fixture).await;
|
||||
assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}");
|
||||
assert!(current_time_millis() < fixture.now_ms + 5_100);
|
||||
assert!(fixture.session.read().await.origin_refresh.in_flight);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
let fixture = post_refresh_terminal_fixture_with_progress("first-http-failure-cutover", true, false).await;
|
||||
advance_post_refresh_fixture_playback(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&fixture.lease_id,
|
||||
18_000,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await;
|
||||
fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms, 12_000);
|
||||
let hits = Arc::new(AtomicUsize::new(0));
|
||||
let handler_hits = Arc::clone(&hits);
|
||||
let router = axum::Router::new().route(
|
||||
"/index.m3u8",
|
||||
axum::routing::get(move || {
|
||||
let handler_hits = Arc::clone(&handler_hits);
|
||||
async move {
|
||||
let status = if handler_hits.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
axum::http::StatusCode::BAD_GATEWAY
|
||||
} else {
|
||||
axum::http::StatusCode::NOT_FOUND
|
||||
};
|
||||
(status, [(axum::http::header::RETRY_AFTER, "60")], "origin failure")
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("local origin binds");
|
||||
let address = listener.local_addr().expect("local origin address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.expect("local origin serves");
|
||||
});
|
||||
let mut request = post_refresh_owner_request(&fixture);
|
||||
request.origin_entry = crate::manifest_fetch::LiveHlsOriginEntry::parse(&format!("http://{address}/index.m3u8"))
|
||||
.expect("local origin URL");
|
||||
let refresh = tokio::spawn(crate::refresh::trigger_origin_refresh_sync(request));
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if fixture.session.read().await.origin_control.path_condition
|
||||
== HlsOriginPathCondition::RetryableFetchFailure
|
||||
&& fixture.ctx.hls_proxy.availability_reevaluations().owner_count() == 1
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("first failed attempt publishes degraded evidence and owns the deadline");
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 1);
|
||||
assert!(fixture.session.read().await.origin_refresh.in_flight);
|
||||
|
||||
let mode = wait_for_terminal_mode(&fixture).await;
|
||||
assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}");
|
||||
assert!(current_time_millis() < fixture.now_ms + 5_100);
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 1, "cutover must precede the capped six-second retry");
|
||||
assert!(fixture.session.read().await.origin_refresh.in_flight);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(8), refresh)
|
||||
.await
|
||||
.expect("refresh finishes")
|
||||
.expect("refresh task"));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovered_origin_cancels_the_retained_reserve_cutover() {
|
||||
let fixture = post_refresh_terminal_fixture_with_progress("recovered-reserve-owner", true, false).await;
|
||||
advance_post_refresh_fixture_playback(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&fixture.lease_id,
|
||||
18_000,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await;
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("live lease");
|
||||
assert_eq!(
|
||||
commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&lease,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await,
|
||||
HlsTerminalResolution::LiveAllowed
|
||||
);
|
||||
assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1);
|
||||
fixture.session.write().await.origin_control.record_media_progress(current_time_millis(), 12_000);
|
||||
fixture.ctx.hls_proxy.availability_reevaluations().notify_session_evidence_changed(&fixture.proxy_session_id);
|
||||
wait_for_availability_owner_completion(&fixture).await;
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("recovered lease");
|
||||
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leaving_a_live_lease() {
|
||||
let fixture = post_refresh_terminal_fixture_with_progress("expired-splice-unavailable", true, false).await;
|
||||
advance_post_refresh_fixture_playback(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&fixture.lease_id,
|
||||
26_000,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await;
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("live lease");
|
||||
assert_eq!(
|
||||
commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&lease,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await,
|
||||
HlsTerminalResolution::Committed
|
||||
);
|
||||
let terminal = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("terminal lease");
|
||||
assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
|
||||
assert_eq!(
|
||||
commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&terminal,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await,
|
||||
HlsTerminalResolution::Committed
|
||||
);
|
||||
assert!(!fixture.session.read().await.has_terminal_tail_protections());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_losing_refresh_work() {
|
||||
let fixture = post_refresh_terminal_fixture_with_progress("reserve-owner-recovery-upgrade", true, false).await;
|
||||
fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms);
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("live lease");
|
||||
assert_eq!(
|
||||
commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&lease,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await,
|
||||
HlsTerminalResolution::LiveAllowed
|
||||
);
|
||||
assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1);
|
||||
assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_late_origin_can_finish_custom_response_after_the_splice_deadline() {
|
||||
let fixture = post_refresh_terminal_fixture_with_progress("publication-late-unavailable", true, false).await;
|
||||
advance_post_refresh_fixture_playback(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&fixture.lease_id,
|
||||
26_000,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await;
|
||||
fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms - 18_000, 12_000);
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
|
||||
.await
|
||||
.expect("live lease");
|
||||
assert_eq!(
|
||||
commit_terminal_tail_if_lease_reserve_requires_cutover(
|
||||
&fixture.ctx,
|
||||
&fixture.session,
|
||||
&fixture.proxy_session_id,
|
||||
&lease,
|
||||
fixture.now_ms,
|
||||
)
|
||||
.await,
|
||||
HlsTerminalResolution::Committed
|
||||
);
|
||||
let lease = fixture
|
||||
.ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("terminal lease");
|
||||
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
|
||||
}
|
||||
|
||||
@@ -63,6 +63,7 @@ pub(super) async fn wait_for_runtime_custom_plan(fixture: &PostRefreshTerminalFi
|
||||
let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode {
|
||||
HlsLeasePlaybackMode::Live => "live",
|
||||
HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail",
|
||||
HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response",
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable",
|
||||
HlsLeasePlaybackMode::Ended => "ended",
|
||||
});
|
||||
@@ -332,7 +333,7 @@ pub(super) async fn assert_multi_lease_fallback_handles_pending_and_unavailable(
|
||||
assert_eq!(aggregate.total, 2);
|
||||
assert_eq!(aggregate.pending_owned, 1);
|
||||
assert_eq!(aggregate.terminal_committed, 1, "{aggregate:?}");
|
||||
assert!(aggregate.unresolved.is_empty());
|
||||
assert_eq!(aggregate.unresolved.len(), 0);
|
||||
assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1);
|
||||
let unavailable = fixture
|
||||
.ctx
|
||||
|
||||
@@ -148,6 +148,7 @@ pub(super) async fn install_terminal_pending_ready_base(
|
||||
host_local_index: 0,
|
||||
},
|
||||
proxy_seq: 40,
|
||||
media_start_ms: 0,
|
||||
duration_ms: asset.duration_ms(),
|
||||
proxy_file_ext: "ts".to_string(),
|
||||
content_type: "video/mp2t".to_string(),
|
||||
@@ -316,7 +317,8 @@ pub(super) async fn assert_terminal_pending_registration_failure_commits_unavail
|
||||
fixture.expected_asset,
|
||||
fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(),
|
||||
failure,
|
||||
);
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(resolution, HlsTerminalResolution::Committed);
|
||||
let lease = fixture
|
||||
@@ -340,6 +342,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l
|
||||
let mut session = fixture.session.write().await;
|
||||
session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict;
|
||||
let origin_epoch = session.origin_control.origin_epoch;
|
||||
let base_end_ms = session.segments[&40].media_start_ms + session.segments[&40].duration_ms;
|
||||
for proxy_seq in 41_u64..=44 {
|
||||
session.segments.insert(
|
||||
proxy_seq,
|
||||
@@ -351,6 +354,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l
|
||||
host_local_index: u32::try_from(proxy_seq.saturating_sub(40)).unwrap_or(u32::MAX),
|
||||
},
|
||||
proxy_seq,
|
||||
media_start_ms: base_end_ms + (proxy_seq - 41) * 20_000,
|
||||
duration_ms: 20_000,
|
||||
proxy_file_ext: "ts".to_string(),
|
||||
content_type: "video/mp2t".to_string(),
|
||||
|
||||
@@ -81,6 +81,8 @@ pub enum HlsAvailabilityReevaluationRegistration {
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum HlsAvailabilityReevaluationMode {
|
||||
RecoveryPressure,
|
||||
/// Tracks a live reserve deadline independently of a client request or a running origin fetch.
|
||||
TerminalReserve,
|
||||
PostRefresh(HlsPostRefreshAvailabilityReason),
|
||||
}
|
||||
|
||||
@@ -90,8 +92,10 @@ impl HlsAvailabilityReevaluationMode {
|
||||
(Self::PostRefresh(current), Self::PostRefresh(incoming)) => {
|
||||
Self::PostRefresh(stronger_post_refresh_reason(current, incoming))
|
||||
}
|
||||
(Self::PostRefresh(reason), Self::RecoveryPressure)
|
||||
| (Self::RecoveryPressure, Self::PostRefresh(reason)) => Self::PostRefresh(reason),
|
||||
(Self::PostRefresh(reason), Self::RecoveryPressure | Self::TerminalReserve)
|
||||
| (Self::RecoveryPressure | Self::TerminalReserve, Self::PostRefresh(reason)) => Self::PostRefresh(reason),
|
||||
(Self::TerminalReserve, Self::RecoveryPressure | Self::TerminalReserve)
|
||||
| (Self::RecoveryPressure, Self::TerminalReserve) => Self::TerminalReserve,
|
||||
(Self::RecoveryPressure, Self::RecoveryPressure) => Self::RecoveryPressure,
|
||||
}
|
||||
}
|
||||
@@ -106,10 +110,14 @@ const fn stronger_post_refresh_reason(
|
||||
| (_, HlsPostRefreshAvailabilityReason::HardManifestFailure) => {
|
||||
HlsPostRefreshAvailabilityReason::HardManifestFailure
|
||||
}
|
||||
(HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, _)
|
||||
| (_, HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict) => {
|
||||
HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict
|
||||
}
|
||||
(
|
||||
HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict,
|
||||
HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict,
|
||||
) => HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict,
|
||||
HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
|
||||
HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
|
||||
) => HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,7 +182,7 @@ impl HlsAvailabilityReevaluationCoordinator {
|
||||
pub fn register<Work, WorkFuture>(
|
||||
self: &Arc<Self>,
|
||||
key: HlsAvailabilityReevaluationOwnerKey,
|
||||
mode: HlsAvailabilityReevaluationMode,
|
||||
mut mode: HlsAvailabilityReevaluationMode,
|
||||
work: Work,
|
||||
) -> HlsAvailabilityReevaluationRegistration
|
||||
where
|
||||
@@ -186,24 +194,25 @@ impl HlsAvailabilityReevaluationCoordinator {
|
||||
};
|
||||
let mut state = self.lock_state();
|
||||
if let Some(current) = state.owners.get_mut(&key.proxy_session_id) {
|
||||
if current.key == key {
|
||||
current.mode = current.mode.merge(mode);
|
||||
current.rerun_requested = true;
|
||||
current.wake.notify_one();
|
||||
return HlsAvailabilityReevaluationRegistration::AlreadyOwned;
|
||||
}
|
||||
if !key.supersedes(¤t.key) {
|
||||
if current.key != key && !key.supersedes(¤t.key) {
|
||||
return HlsAvailabilityReevaluationRegistration::Superseded;
|
||||
}
|
||||
let installs_refresh_payload = current.mode == HlsAvailabilityReevaluationMode::TerminalReserve
|
||||
&& mode != HlsAvailabilityReevaluationMode::TerminalReserve;
|
||||
mode = current.mode.merge(mode);
|
||||
// A newer demand for the same proxy session is owned by the
|
||||
// already-counted task. The worker will snapshot current session
|
||||
// evidence and atomically rekey itself without acquiring another
|
||||
// permit. Keeping one dirty bit preserves a bounded successor
|
||||
// cycle while coalescing repeated registrations.
|
||||
current.mode = current.mode.merge(mode);
|
||||
current.rerun_requested = true;
|
||||
current.wake.notify_one();
|
||||
return HlsAvailabilityReevaluationRegistration::AlreadyOwned;
|
||||
// A reserve-only worker has no origin request payload. Replace it
|
||||
// when recovery work arrives so merging the mode cannot discard that work.
|
||||
if !installs_refresh_payload {
|
||||
current.mode = mode;
|
||||
current.rerun_requested = true;
|
||||
current.wake.notify_one();
|
||||
return HlsAvailabilityReevaluationRegistration::AlreadyOwned;
|
||||
}
|
||||
}
|
||||
let Ok(task_permit) = Arc::clone(&self.task_capacity).try_acquire_owned() else {
|
||||
return HlsAvailabilityReevaluationRegistration::CapacityExceeded;
|
||||
|
||||
@@ -48,6 +48,8 @@ pub struct StagedCacheObject {
|
||||
pub size: u64,
|
||||
cache_path_generation: Arc<CachePathGeneration>,
|
||||
_registration: Arc<ActiveTempFileRegistration>,
|
||||
pub(crate) write_budget: Option<Arc<HlsRevisionDiskReservation>>,
|
||||
cleanup: Arc<StagedFileCleanup>,
|
||||
}
|
||||
|
||||
/// Typed source used to distinguish a decoded-object size violation from a filesystem failure.
|
||||
@@ -106,6 +108,10 @@ pub struct HlsSegmentCache {
|
||||
capacity_reclaimer: StdRwLock<Option<Weak<dyn HlsCacheCapacityReclaimer>>>,
|
||||
capacity_admission_gate: AsyncMutex<()>,
|
||||
owned_operation_permits: Arc<Semaphore>,
|
||||
#[cfg(test)]
|
||||
startup_test_checkpoint: StdMutex<Option<test_support::StartupWriteCheckpoint>>,
|
||||
#[cfg(test)]
|
||||
delete_test_failures: StdMutex<HashMap<PathBuf, usize>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -125,8 +131,13 @@ struct CacheCapacityState {
|
||||
revision: Arc<CapacityRevision>,
|
||||
active_mutations: HashSet<PathBuf>,
|
||||
revision_reservations: HashMap<u64, (String, u64)>,
|
||||
/// Revision file bytes found by the last completed usage scan.
|
||||
next_write_id: u64,
|
||||
write_owners: HashMap<PathBuf, u64>,
|
||||
abandoned_writes: HashMap<PathBuf, (u64, String)>,
|
||||
reserved_spools: HashMap<PathBuf, u64>,
|
||||
/// Current revision bytes, reconciled by scans and successful file mutations.
|
||||
revision_bytes: u64,
|
||||
revision_metrics: Option<Weak<crate::HlsCacheMetrics>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -135,17 +146,24 @@ struct CapacityRevision;
|
||||
#[derive(Default)]
|
||||
struct TempFileState {
|
||||
active_files: HashSet<PathBuf>,
|
||||
revision: Arc<CapacityRevision>,
|
||||
deletion_reservations: HashSet<PathBuf>,
|
||||
}
|
||||
|
||||
struct ActiveTempFileRegistration {
|
||||
path: PathBuf,
|
||||
state: Arc<StdMutex<TempFileState>>,
|
||||
changed: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl Drop for ActiveTempFileRegistration {
|
||||
fn drop(&mut self) {
|
||||
self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_files.remove(&self.path);
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.active_files.remove(&self.path) {
|
||||
state.revision = Arc::new(CapacityRevision);
|
||||
drop(state);
|
||||
self.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +189,8 @@ struct CapacityMutationReservation {
|
||||
session_component: String,
|
||||
replacement: Option<(u64, u64)>,
|
||||
filesystem_mutation_started: bool,
|
||||
write_budget: Option<Arc<HlsRevisionDiskReservation>>,
|
||||
transferred_bytes: u64,
|
||||
capacity: Arc<StdMutex<CacheCapacityState>>,
|
||||
changed: Arc<Notify>,
|
||||
}
|
||||
@@ -198,6 +218,8 @@ pub struct HlsRevisionDiskReservation {
|
||||
changed: Arc<Notify>,
|
||||
/// Set when bytes were written under this reservation outside the staged commit accounting.
|
||||
unaccounted_writes: bool,
|
||||
generation: Arc<CachePathGeneration>,
|
||||
cache_path: PathBuf,
|
||||
}
|
||||
|
||||
pub struct HlsRevisionFilePin {
|
||||
@@ -217,6 +239,9 @@ mod lifecycle;
|
||||
mod revisions;
|
||||
mod staging;
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
mod write_budget;
|
||||
#[cfg(test)]
|
||||
use capacity::CapacityReservationError;
|
||||
#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")]
|
||||
pub use capacity::{
|
||||
@@ -226,3 +251,6 @@ pub use capacity::{
|
||||
pub use error::{hls_cache_capacity_from_io, hls_cache_object_limit_from_io};
|
||||
pub use keys::HlsCacheObjectKey;
|
||||
pub use staging::CachedSegmentMetadata;
|
||||
#[cfg(test)]
|
||||
pub(crate) use test_support::StartupWritePhase;
|
||||
use write_budget::StagedFileCleanup;
|
||||
|
||||
Vendored
+168
-22
@@ -1,7 +1,7 @@
|
||||
use super::{
|
||||
error::{cache_object_limit_error, capacity_error},
|
||||
hls_cache_capacity_from_io,
|
||||
lifecycle::{is_temp_cache_file, REWRITE_SECRET_FINGERPRINT_FILE},
|
||||
lifecycle::REWRITE_SECRET_FINGERPRINT_FILE,
|
||||
revisions::{is_revision_cache_path, revision_reserved_bytes},
|
||||
safe_proxy_session_id, CacheCapacityPressure, CacheCapacityState, CapacityInvalidationGuard,
|
||||
CapacityMutationReservation, CapacityRevision, HlsCacheCapacityRevision, HlsCacheObjectKey, HlsSegmentCache,
|
||||
@@ -82,11 +82,21 @@ impl CapacityMutationReservation {
|
||||
return Err(CapacityReservationError::Invalidated);
|
||||
}
|
||||
let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
|
||||
let projected_total = capacity.total_bytes.saturating_sub(old_size).saturating_add(new_size);
|
||||
let projected_session = session_size.saturating_sub(old_size).saturating_add(new_size);
|
||||
// Old bytes remain on disk until rename succeeds. Reserve growth now,
|
||||
// but publish any shrink only after the filesystem mutation completes.
|
||||
let growth = new_size.saturating_sub(old_size);
|
||||
let projected_total = capacity.total_bytes.saturating_add(growth);
|
||||
let projected_session = session_size.saturating_add(growth);
|
||||
let (reserved_global, reserved_session) = revision_reserved_bytes(&capacity, &self.session_component);
|
||||
if projected_total.saturating_add(reserved_global) > max_cache_bytes
|
||||
|| projected_session.saturating_add(reserved_session) > max_session_bytes
|
||||
let transferred = self
|
||||
.write_budget
|
||||
.as_ref()
|
||||
.and_then(|budget| {
|
||||
capacity.revision_reservations.get(&budget.id).filter(|(owner, _)| owner == &self.session_component)
|
||||
})
|
||||
.map_or(0, |(_, bytes)| (*bytes).min(new_size.saturating_sub(old_size)));
|
||||
if projected_total.saturating_add(reserved_global.saturating_sub(transferred)) > max_cache_bytes
|
||||
|| projected_session.saturating_add(reserved_session.saturating_sub(transferred)) > max_session_bytes
|
||||
{
|
||||
return Err(CapacityReservationError::Exceeded {
|
||||
pressure: CacheCapacityPressure {
|
||||
@@ -96,10 +106,10 @@ impl CapacityMutationReservation {
|
||||
current_global_bytes: capacity.total_bytes,
|
||||
staged_bytes: new_size,
|
||||
required_session_bytes: projected_session
|
||||
.saturating_add(reserved_session)
|
||||
.saturating_add(reserved_session.saturating_sub(transferred))
|
||||
.saturating_sub(max_session_bytes),
|
||||
required_global_bytes: projected_total
|
||||
.saturating_add(reserved_global)
|
||||
.saturating_add(reserved_global.saturating_sub(transferred))
|
||||
.saturating_sub(max_cache_bytes),
|
||||
},
|
||||
// Capture the opaque token in the same critical section as the
|
||||
@@ -108,6 +118,12 @@ impl CapacityMutationReservation {
|
||||
revision: HlsCacheCapacityRevision(Arc::clone(&capacity.revision)),
|
||||
});
|
||||
}
|
||||
if let Some(budget) = &self.write_budget {
|
||||
if let Some((_, bytes)) = capacity.revision_reservations.get_mut(&budget.id) {
|
||||
*bytes -= transferred;
|
||||
}
|
||||
}
|
||||
self.transferred_bytes = transferred;
|
||||
capacity.total_bytes = projected_total;
|
||||
store_session_bytes(&mut capacity, &self.session_component, projected_session);
|
||||
self.replacement = Some((old_size, new_size));
|
||||
@@ -117,12 +133,36 @@ impl CapacityMutationReservation {
|
||||
pub(super) fn mark_filesystem_mutation_started(&mut self) { self.filesystem_mutation_started = true; }
|
||||
|
||||
pub(super) fn finish_replacement(mut self) {
|
||||
let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some((old, new)) = self.replacement {
|
||||
let released = old.saturating_sub(new);
|
||||
capacity.total_bytes = capacity.total_bytes.saturating_sub(released);
|
||||
let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
|
||||
store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(released));
|
||||
if super::revisions::is_revision_cache_path(&self.path) {
|
||||
capacity.revision_bytes = capacity.revision_bytes.saturating_sub(old).saturating_add(new);
|
||||
}
|
||||
}
|
||||
capacity.abandoned_writes.remove(&self.path);
|
||||
if let Some(budget) = &self.write_budget {
|
||||
capacity.write_owners.insert(self.path.clone(), budget.id);
|
||||
} else {
|
||||
capacity.write_owners.remove(&self.path);
|
||||
}
|
||||
drop(capacity);
|
||||
self.replacement = None;
|
||||
self.finish_mutation(None);
|
||||
}
|
||||
|
||||
pub(super) fn finish_delete(mut self, deleted_size: u64) {
|
||||
self.finish_mutation(Some(deleted_size));
|
||||
let unaccounted = self
|
||||
.capacity
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.reserved_spools
|
||||
.remove(&self.path)
|
||||
.is_some();
|
||||
self.finish_mutation(Some(if unaccounted { 0 } else { deleted_size }));
|
||||
self.replacement = None;
|
||||
}
|
||||
|
||||
@@ -134,11 +174,17 @@ impl CapacityMutationReservation {
|
||||
capacity.active_mutations.remove(&self.path);
|
||||
if capacity.initialized && capacity.cache_path == self.cache_path {
|
||||
if let Some(deleted_size) = deleted_size {
|
||||
capacity.write_owners.remove(&self.path);
|
||||
capacity.abandoned_writes.remove(&self.path);
|
||||
if super::revisions::is_revision_cache_path(&self.path) {
|
||||
capacity.revision_bytes = capacity.revision_bytes.saturating_sub(deleted_size);
|
||||
}
|
||||
capacity.total_bytes = capacity.total_bytes.saturating_sub(deleted_size);
|
||||
let session_bytes = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
|
||||
store_session_bytes(&mut capacity, &self.session_component, session_bytes.saturating_sub(deleted_size));
|
||||
}
|
||||
}
|
||||
update_revision_metrics(&capacity);
|
||||
capacity.revision = Arc::new(CapacityRevision);
|
||||
drop(capacity);
|
||||
self.changed.notify_waiters();
|
||||
@@ -152,18 +198,22 @@ impl Drop for CapacityMutationReservation {
|
||||
return;
|
||||
}
|
||||
capacity.active_mutations.remove(&self.path);
|
||||
if !self.filesystem_mutation_started && self.transferred_bytes > 0 {
|
||||
if let Some((_, bytes)) =
|
||||
self.write_budget.as_ref().and_then(|budget| capacity.revision_reservations.get_mut(&budget.id))
|
||||
{
|
||||
*bytes = bytes.saturating_add(self.transferred_bytes);
|
||||
}
|
||||
}
|
||||
let revision_changed = if self.filesystem_mutation_started {
|
||||
capacity.initialized = false;
|
||||
true
|
||||
} else if capacity.initialized && capacity.cache_path == self.cache_path {
|
||||
if let Some((old_size, new_size)) = self.replacement.take() {
|
||||
capacity.total_bytes = capacity.total_bytes.saturating_sub(new_size).saturating_add(old_size);
|
||||
let growth = new_size.saturating_sub(old_size);
|
||||
capacity.total_bytes = capacity.total_bytes.saturating_sub(growth);
|
||||
let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
|
||||
store_session_bytes(
|
||||
&mut capacity,
|
||||
&self.session_component,
|
||||
session_size.saturating_sub(new_size).saturating_add(old_size),
|
||||
);
|
||||
store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(growth));
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -342,13 +392,36 @@ impl HlsSegmentCache {
|
||||
notified.await;
|
||||
continue;
|
||||
};
|
||||
let (total_bytes, session_bytes, revision_bytes) = scan_committed_cache_usage(cache_path).await?;
|
||||
let (temp_revision, mut excluded) = {
|
||||
let temps = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let excluded: std::collections::HashSet<_> = temps
|
||||
.active_files
|
||||
.iter()
|
||||
.filter(|path| super::lifecycle::is_temp_cache_file(path))
|
||||
.cloned()
|
||||
.collect();
|
||||
(Arc::clone(&temps.revision), excluded)
|
||||
};
|
||||
excluded.extend(
|
||||
self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reserved_spools.keys().cloned(),
|
||||
);
|
||||
#[cfg(test)]
|
||||
self.startup_write_checkpoint(super::StartupWritePhase::AfterInventorySnapshot).await?;
|
||||
let (total_bytes, session_bytes, revision_bytes) =
|
||||
scan_cache_usage_excluding(cache_path, &excluded).await?;
|
||||
if self.cache_path_snapshot().path != cache_path {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during usage scan"));
|
||||
}
|
||||
// Registration precedes file creation. Changes after this check cannot
|
||||
// have affected the completed scan; no nested temp/capacity locks are needed.
|
||||
let temps_changed = !Arc::ptr_eq(
|
||||
&self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).revision,
|
||||
&temp_revision,
|
||||
);
|
||||
let scan_invalidated = {
|
||||
let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !Arc::ptr_eq(&capacity.revision, &revision)
|
||||
if temps_changed
|
||||
|| !Arc::ptr_eq(&capacity.revision, &revision)
|
||||
|| capacity.active_mutations.iter().any(|path| path.starts_with(cache_path))
|
||||
{
|
||||
true
|
||||
@@ -358,6 +431,18 @@ impl HlsSegmentCache {
|
||||
capacity.total_bytes = total_bytes;
|
||||
capacity.session_bytes = session_bytes;
|
||||
capacity.revision_bytes = revision_bytes;
|
||||
let sessions: std::collections::HashSet<_> = capacity.session_bytes.keys().cloned().collect();
|
||||
let active_owners: std::collections::HashSet<_> =
|
||||
capacity.revision_reservations.keys().copied().collect();
|
||||
capacity.write_owners.retain(|path, owner| {
|
||||
active_owners.contains(owner)
|
||||
|| path.strip_prefix(cache_path).ok().and_then(|path| path.components().next()).is_some_and(
|
||||
|session| sessions.contains(session.as_os_str().to_string_lossy().as_ref()),
|
||||
)
|
||||
});
|
||||
let owners = capacity.write_owners.clone();
|
||||
capacity.abandoned_writes.retain(|path, (owner, _)| owners.get(path) == Some(owner));
|
||||
update_revision_metrics(&capacity);
|
||||
false
|
||||
}
|
||||
};
|
||||
@@ -407,6 +492,8 @@ impl HlsSegmentCache {
|
||||
session_component,
|
||||
replacement: None,
|
||||
filesystem_mutation_started: false,
|
||||
write_budget: None,
|
||||
transferred_bytes: 0,
|
||||
capacity: Arc::clone(&self.capacity),
|
||||
changed: Arc::clone(&self.capacity_changed),
|
||||
})
|
||||
@@ -421,9 +508,39 @@ impl HlsSegmentCache {
|
||||
max_cache_bytes: u64,
|
||||
max_session_bytes: u64,
|
||||
) -> io::Result<CapacityMutationReservation> {
|
||||
self.prepare_capacity_replacement_with_budget(
|
||||
cache_path,
|
||||
final_path,
|
||||
session_component,
|
||||
new_size,
|
||||
max_cache_bytes,
|
||||
max_session_bytes,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn prepare_capacity_replacement_with_budget(
|
||||
&self,
|
||||
cache_path: &Path,
|
||||
final_path: &Path,
|
||||
session_component: String,
|
||||
new_size: u64,
|
||||
max_cache_bytes: u64,
|
||||
max_session_bytes: u64,
|
||||
budget: Option<Arc<super::HlsRevisionDiskReservation>>,
|
||||
) -> io::Result<CapacityMutationReservation> {
|
||||
if budget.as_ref().is_some_and(|budget| {
|
||||
!Arc::ptr_eq(&budget.capacity, &self.capacity)
|
||||
|| !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation)
|
||||
}) {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation"));
|
||||
}
|
||||
loop {
|
||||
self.ensure_capacity_initialized(cache_path).await?;
|
||||
let mut reservation = self.begin_capacity_mutation(cache_path, final_path, session_component.clone()).await;
|
||||
reservation.write_budget = budget.clone();
|
||||
let old_size = match fs::metadata(final_path).await {
|
||||
Ok(metadata) => metadata.len(),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => 0,
|
||||
@@ -465,7 +582,15 @@ fn invalidate_capacity_state(capacity: &StdMutex<CacheCapacityState>, changed: &
|
||||
changed.notify_waiters();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<(u64, HashMap<String, u64>, u64)> {
|
||||
scan_cache_usage_excluding(cache_path, &std::collections::HashSet::new()).await
|
||||
}
|
||||
|
||||
async fn scan_cache_usage_excluding(
|
||||
cache_path: &Path,
|
||||
excluded: &std::collections::HashSet<PathBuf>,
|
||||
) -> io::Result<(u64, HashMap<String, u64>, u64)> {
|
||||
let mut total_bytes = 0_u64;
|
||||
let mut revision_bytes = 0_u64;
|
||||
let mut session_bytes = HashMap::new();
|
||||
@@ -475,22 +600,37 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
while let Some(root_entry) = root_entries.next_entry().await? {
|
||||
if !root_entry.file_type().await?.is_dir() {
|
||||
continue;
|
||||
match root_entry.file_type().await {
|
||||
Ok(kind) if kind.is_dir() => {}
|
||||
Ok(_) => continue,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
let session_component = root_entry.file_name().to_string_lossy().into_owned();
|
||||
let mut bytes = 0_u64;
|
||||
let mut pending_dirs = vec![root_entry.path()];
|
||||
while let Some(dir) = pending_dirs.pop() {
|
||||
let mut entries = fs::read_dir(dir).await?;
|
||||
let mut entries = match fs::read_dir(dir).await {
|
||||
Ok(entries) => entries,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let file_type = entry.file_type().await?;
|
||||
let file_type = match entry.file_type().await {
|
||||
Ok(kind) => kind,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if file_type.is_dir() {
|
||||
pending_dirs.push(entry.path());
|
||||
} else if file_type.is_file() {
|
||||
let revision = is_revision_cache_path(&entry.path());
|
||||
if revision || !is_temp_cache_file(&entry.path()) {
|
||||
let size = entry.metadata().await?.len();
|
||||
if !excluded.contains(&entry.path()) {
|
||||
let size = match entry.metadata().await {
|
||||
Ok(metadata) => metadata.len(),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
bytes = bytes.saturating_add(size);
|
||||
if revision {
|
||||
revision_bytes = revision_bytes.saturating_add(size);
|
||||
@@ -506,3 +646,9 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<
|
||||
}
|
||||
Ok((total_bytes, session_bytes, revision_bytes))
|
||||
}
|
||||
|
||||
pub(super) fn update_revision_metrics(capacity: &CacheCapacityState) {
|
||||
if let Some(metrics) = capacity.revision_metrics.as_ref().and_then(Weak::upgrade) {
|
||||
metrics.record_revision_disk_bytes(capacity.revision_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+30
-12
@@ -1,9 +1,9 @@
|
||||
use super::{
|
||||
revisions::is_revision_cache_path, staging::run_owned_cache_operation, ActiveTempFileRegistration,
|
||||
CacheCapacityState, CacheInvalidationOutcome, CachePathDeletionReservation, CachePathGeneration, CachePathState,
|
||||
CachedSegmentMetadata, CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer,
|
||||
HlsCacheCapacityRevision, HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState,
|
||||
DEFAULT_HLS_CACHE_PATH, MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN,
|
||||
staging::run_owned_cache_operation, ActiveTempFileRegistration, CacheCapacityState, CacheInvalidationOutcome,
|
||||
CachePathDeletionReservation, CachePathGeneration, CachePathState, CachedSegmentMetadata,
|
||||
CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer, HlsCacheCapacityRevision,
|
||||
HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState, DEFAULT_HLS_CACHE_PATH,
|
||||
MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN,
|
||||
};
|
||||
use log::warn;
|
||||
use std::{
|
||||
@@ -41,6 +41,10 @@ impl HlsSegmentCache {
|
||||
capacity_reclaimer: StdRwLock::new(None),
|
||||
capacity_admission_gate: AsyncMutex::new(()),
|
||||
owned_operation_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS)),
|
||||
#[cfg(test)]
|
||||
startup_test_checkpoint: StdMutex::new(None),
|
||||
#[cfg(test)]
|
||||
delete_test_failures: StdMutex::new(std::collections::HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,6 +127,7 @@ impl HlsSegmentCache {
|
||||
|
||||
pub async fn delete<K: HlsCacheObjectKey>(&self, key: &K) -> io::Result<()> {
|
||||
let cache_path = self.cache_path_snapshot();
|
||||
self.ensure_capacity_initialized(&cache_path.path).await?;
|
||||
let path = Self::path_for_key_in(&cache_path.path, key);
|
||||
let reservation = self.begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()).await;
|
||||
self.delete_with_reservation(path, reservation).await
|
||||
@@ -130,6 +135,7 @@ impl HlsSegmentCache {
|
||||
|
||||
pub async fn delete_if_inactive<K: HlsCacheObjectKey>(&self, key: &K) -> io::Result<()> {
|
||||
let cache_path = self.cache_path_snapshot();
|
||||
self.ensure_capacity_initialized(&cache_path.path).await?;
|
||||
let path = Self::path_for_key_in(&cache_path.path, key);
|
||||
let reservation = self
|
||||
.try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component())
|
||||
@@ -142,6 +148,8 @@ impl HlsSegmentCache {
|
||||
path: PathBuf,
|
||||
reservation: CapacityMutationReservation,
|
||||
) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
self.check_cache_delete_test_failure(&path)?;
|
||||
run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "delete", async move {
|
||||
let size = match fs::metadata(&path).await {
|
||||
Ok(metadata) => metadata.len(),
|
||||
@@ -190,9 +198,7 @@ impl HlsSegmentCache {
|
||||
let cache_path = self.cache_path_snapshot();
|
||||
let candidates = old_temp_files(&cache_path.path, cutoff, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN).await?;
|
||||
let mut deleted = 0_usize;
|
||||
let mut removed_revision = false;
|
||||
for path in candidates {
|
||||
let revision = is_revision_cache_path(&path);
|
||||
let Some(deletion) = self.reserve_path_deletion(&path) else {
|
||||
continue;
|
||||
};
|
||||
@@ -209,14 +215,14 @@ impl HlsSegmentCache {
|
||||
match result {
|
||||
Ok(()) => {
|
||||
deleted = deleted.saturating_add(1);
|
||||
removed_revision |= revision;
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => warn!("HLS temporary cache file cleanup deferred: error_kind={:?}", error.kind()),
|
||||
}
|
||||
}
|
||||
if removed_revision {
|
||||
if deleted > 0 {
|
||||
drop(self.capacity_invalidation_guard());
|
||||
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?;
|
||||
}
|
||||
Ok(deleted)
|
||||
}
|
||||
@@ -238,7 +244,8 @@ impl HlsSegmentCache {
|
||||
drop(deletion);
|
||||
result
|
||||
})
|
||||
.await
|
||||
.await?;
|
||||
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await
|
||||
}
|
||||
|
||||
pub async fn delete_orphan_session_dirs(
|
||||
@@ -307,6 +314,9 @@ impl HlsSegmentCache {
|
||||
}
|
||||
}
|
||||
}
|
||||
if removed > 0 {
|
||||
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?;
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
@@ -389,7 +399,8 @@ impl HlsSegmentCache {
|
||||
drop(deletion);
|
||||
result
|
||||
})
|
||||
.await
|
||||
.await?;
|
||||
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await
|
||||
}
|
||||
|
||||
pub async fn read_rewrite_secret_fingerprint(&self) -> io::Result<Option<String>> {
|
||||
@@ -422,7 +433,14 @@ impl HlsSegmentCache {
|
||||
if !state.active_files.insert(path.to_path_buf()) {
|
||||
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "hls cache temp path is already active"));
|
||||
}
|
||||
Ok(Arc::new(ActiveTempFileRegistration { path: path.to_path_buf(), state: Arc::clone(&self.temp_files) }))
|
||||
state.revision = Arc::new(CapacityRevision);
|
||||
drop(state);
|
||||
self.capacity_changed.notify_waiters();
|
||||
Ok(Arc::new(ActiveTempFileRegistration {
|
||||
path: path.to_path_buf(),
|
||||
state: Arc::clone(&self.temp_files),
|
||||
changed: Arc::clone(&self.capacity_changed),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn reserve_path_deletion(&self, path: &Path) -> Option<CachePathDeletionReservation> {
|
||||
|
||||
@@ -1,5 +1,81 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn temp_registration_and_last_owner_release_advance_inventory_revision() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
let initial = Arc::clone(&cache.temp_files.lock().unwrap().revision);
|
||||
let registration = cache.register_active_temp_path(&directory.path().join("segment.ts.tmp.test"))?;
|
||||
let registered = Arc::clone(&cache.temp_files.lock().unwrap().revision);
|
||||
assert!(!Arc::ptr_eq(&initial, ®istered));
|
||||
let last_owner = Arc::clone(®istration);
|
||||
drop(registration);
|
||||
assert!(Arc::ptr_eq(®istered, &cache.temp_files.lock().unwrap().revision));
|
||||
drop(last_owner);
|
||||
assert!(!Arc::ptr_eq(®istered, &cache.temp_files.lock().unwrap().revision));
|
||||
assert!(cache.temp_files.lock().unwrap().active_files.is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attaching_a_budget_to_shared_staging_returns_an_error_and_preserves_its_owner() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
let key = SegmentCacheKey::new(ProxySessionId("shared-staging".into()), 0, "ts");
|
||||
let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?);
|
||||
let staged = cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
b"test".as_slice(),
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(Arc::clone(&budget)),
|
||||
)
|
||||
.await?;
|
||||
let error = staged.clone().with_write_budget(None).unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
||||
assert_eq!(tokio::fs::read(&staged.path).await?, b"test");
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20);
|
||||
cache.commit_staged(&key, staged).await?;
|
||||
drop(budget);
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 4);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inventory_scan_retries_when_a_reserved_temp_file_is_registered_after_its_snapshot() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = Arc::new(HlsSegmentCache::with_cache_path(directory.path()));
|
||||
cache.update_cache_limits(60, 60);
|
||||
let key = SegmentCacheKey::new(ProxySessionId("inventory-race".into()), 0, "ts");
|
||||
cache.write_bytes_and_commit(&key, &[1; 20]).await?;
|
||||
let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?);
|
||||
cache.invalidate_capacity_accounting();
|
||||
let (reached, resume) = cache.pause_startup_write(StartupWritePhase::AfterInventorySnapshot);
|
||||
let scan_cache = Arc::clone(&cache);
|
||||
let id = key.proxy_session_id().clone();
|
||||
let scan = tokio::spawn(async move { scan_cache.capacity_usage(&id).await });
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), reached).await.unwrap().unwrap();
|
||||
let staged = cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
&[2; 20][..],
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(Arc::clone(&budget)),
|
||||
)
|
||||
.await?;
|
||||
resume.send(()).unwrap();
|
||||
let usage = tokio::time::timeout(std::time::Duration::from_secs(2), scan).await.unwrap().unwrap()?;
|
||||
cache.remove_staged(staged).await?;
|
||||
drop(budget);
|
||||
assert_eq!(usage.global_bytes, 20, "reserved staging bytes must not enter the committed inventory");
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 20);
|
||||
let next = cache.reserve_revision_peak(&key, 30).await?;
|
||||
drop(next);
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn revision_peak_and_legacy_cache_share_the_global_and_session_quota() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
@@ -54,3 +130,375 @@ async fn processed_revision_retries_and_ranges_preserve_bytes_across_cache_repla
|
||||
assert_eq!(cache.delete_orphan_revision_files(SystemTime::now()).await?, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// These are the two real disk-cache transitions from debug/bug.log. Sparse
|
||||
// fixtures preserve the exact byte counts without allocating a GiB of RAM/disk.
|
||||
#[tokio::test]
|
||||
async fn logged_normal_commit_consumes_its_own_peak_reservation() -> io::Result<()> {
|
||||
logged_peak_case(9_994_832).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn logged_revision_publication_consumes_its_remaining_peak_reservation() -> io::Result<()> {
|
||||
logged_peak_case(8_052_416).await
|
||||
}
|
||||
|
||||
async fn logged_peak_case(size: u64) -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
cache.update_cache_limits(1_073_741_824, 1_073_741_824);
|
||||
let key = SegmentCacheKey::new(ProxySessionId("logged-session".into()), 84, "ts");
|
||||
let existing = SegmentCacheKey::new(key.proxy_session_id().clone(), 0, "ts");
|
||||
let path = cache.object_path(&existing);
|
||||
tokio::fs::create_dir_all(path.parent().unwrap()).await?;
|
||||
let file = tokio::fs::File::create(path).await?;
|
||||
file.set_len(1_040_683_024).await?;
|
||||
drop(file);
|
||||
let reservation = std::sync::Arc::new(cache.reserve_revision_peak(&key, 3 * size).await?);
|
||||
let staged_path = cache.object_path(&key).with_extension("ts.tmp.regression");
|
||||
let file = tokio::fs::File::create(&staged_path).await?;
|
||||
file.set_len(size).await?;
|
||||
drop(file);
|
||||
let staged = cache.adopt_staged_file(staged_path, size)?.with_write_budget(Some(reservation.clone()))?;
|
||||
let committed = cache.commit_staged(&key, staged).await?;
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let revision = store.create(key.proxy_session_id().clone(), 84, crate::SegmentRevisionKind::Processed)?;
|
||||
let published = cache
|
||||
.publish_processed_revision_with_budget(
|
||||
&revision.revision().key,
|
||||
&committed.path,
|
||||
Instant::now() + std::time::Duration::from_secs(5),
|
||||
Some(reservation.clone()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(published.size, size);
|
||||
drop(reservation);
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 1_040_683_024 + 2 * size);
|
||||
assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn processed_reservation_release_wakes_before_and_after_wait_registration() -> io::Result<()> {
|
||||
for registered in [false, true] {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
let key = SegmentCacheKey::new(ProxySessionId("release".into()), 0, "ts");
|
||||
let reservation = cache.reserve_revision_peak(&key, 30).await?;
|
||||
assert!(!reservation.unaccounted_writes);
|
||||
let revision = cache.capacity_revision();
|
||||
let waiting = cache.wait_for_capacity_change(&revision);
|
||||
tokio::pin!(waiting);
|
||||
if registered {
|
||||
assert!(futures::poll!(waiting.as_mut()).is_pending());
|
||||
}
|
||||
drop(reservation);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), waiting)
|
||||
.await
|
||||
.expect("releasing only a processed reservation must wake the waiter");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_peak_owners_cannot_consume_each_others_session_or_global_budget() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = std::sync::Arc::new(HlsSegmentCache::with_cache_path(directory.path()));
|
||||
cache.update_cache_limits(180, 120);
|
||||
let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(4));
|
||||
let mut tasks = Vec::new();
|
||||
for (session, seq) in [("one", 0), ("one", 1), ("two", 0)] {
|
||||
let cache = cache.clone();
|
||||
let barrier = barrier.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let key = SegmentCacheKey::new(ProxySessionId(session.into()), seq, "ts");
|
||||
let peak = std::sync::Arc::new(cache.reserve_revision_peak(&key, 60).await?);
|
||||
barrier.wait().await;
|
||||
let staged = cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
&[7; 20][..],
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(peak.clone()),
|
||||
)
|
||||
.await?;
|
||||
let metadata = cache.commit_staged(&key, staged).await?;
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let owner = store.create(key.proxy_session_id().clone(), seq, crate::SegmentRevisionKind::Processed)?;
|
||||
cache
|
||||
.publish_processed_revision_with_budget(
|
||||
&owner.revision().key,
|
||||
&metadata.path,
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(peak),
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, io::Error>(())
|
||||
}));
|
||||
}
|
||||
barrier.wait().await;
|
||||
for task in tasks {
|
||||
task.await??;
|
||||
}
|
||||
let usage = cache.capacity_usage(&ProxySessionId("one".into())).await?;
|
||||
assert_eq!(usage, HlsCacheCapacityUsage { session_bytes: 80, global_bytes: 120 });
|
||||
assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("one".into())), 0);
|
||||
assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("two".into())), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn old_write_cleanup_cannot_delete_a_newer_fetch_attempt() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
let key = SegmentCacheKey::new(ProxySessionId("replace".into()), 0, "ts");
|
||||
let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?);
|
||||
let staged = cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
b"old".as_slice(),
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(old.clone()),
|
||||
)
|
||||
.await?;
|
||||
cache.commit_staged(&key, staged).await?;
|
||||
let new = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?);
|
||||
let staged = cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
b"new".as_slice(),
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(new.clone()),
|
||||
)
|
||||
.await?;
|
||||
cache.commit_staged(&key, staged).await?;
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
|
||||
let error = cache
|
||||
.publish_processed_revision_with_budget(
|
||||
&revision.revision().key,
|
||||
&cache.object_path(&key),
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(old.clone()),
|
||||
)
|
||||
.await
|
||||
.expect_err("an old startup attempt cannot publish a replacement owned by a newer attempt");
|
||||
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
|
||||
assert!(cache.metadata(&revision.revision().key).await?.is_none());
|
||||
cache.delete_write_owned(&key, &old).await?;
|
||||
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new");
|
||||
cache.delete_write_owned(&key, &new).await?;
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pinned_revisions_remain_accounted_after_inventory_reconciliation_and_deletion() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
let metrics = std::sync::Arc::new(crate::HlsCacheMetrics::default());
|
||||
cache.install_revision_metrics(&metrics);
|
||||
let key = SegmentCacheKey::new(ProxySessionId("pinned-scan".into()), 0, "ts");
|
||||
cache.write_bytes_and_commit(&key, &[1; 20]).await?;
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let owner = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
|
||||
let pin = cache.pin_revision_file(&cache.object_path(&owner.revision().key))?;
|
||||
*owner.revision().file_pin.lock().map_err(|_| io::Error::other("file pin"))? = Some(pin);
|
||||
cache
|
||||
.publish_processed_revision(
|
||||
&owner.revision().key,
|
||||
&cache.object_path(&key),
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
)
|
||||
.await?;
|
||||
drop(cache.capacity_invalidation_guard());
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 40);
|
||||
assert_eq!(metrics.snapshot().revision_disk_bytes, cache.scan_revision_disk_bytes().await?);
|
||||
drop(owner);
|
||||
cache.delete_session_dir(key.proxy_session_id()).await?;
|
||||
assert_eq!(metrics.snapshot().revision_disk_bytes, 0);
|
||||
assert_eq!(cache.scan_revision_disk_bytes().await?, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_partial_commit_cleanup_retries_without_deleting_a_replacement() -> io::Result<()> {
|
||||
for replace in [false, true] {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
let key = SegmentCacheKey::new(ProxySessionId("cleanup-retry".into()), 0, "ts");
|
||||
let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?);
|
||||
let staged = cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
b"old".as_slice(),
|
||||
Instant::now() + std::time::Duration::from_secs(2),
|
||||
Some(old.clone()),
|
||||
)
|
||||
.await?;
|
||||
cache.commit_staged(&key, staged).await?;
|
||||
cache.fail_next_cache_deletes(cache.object_path(&key), 1);
|
||||
assert!(cache.delete_write_owned(&key, &old).await.is_err());
|
||||
drop(old);
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 3);
|
||||
if replace {
|
||||
cache.write_bytes_and_commit(&key, b"new").await?;
|
||||
}
|
||||
cache.retry_abandoned_write_deletions().await;
|
||||
if replace {
|
||||
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new");
|
||||
} else {
|
||||
assert!(cache.metadata(&key).await?.is_none());
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0);
|
||||
}
|
||||
cache.retry_abandoned_write_deletions().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_staging_keeps_its_peak_until_pending_file_io_is_joined() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
cache.update_cache_limits(20, 20);
|
||||
let key = SegmentCacheKey::new(ProxySessionId("pending-io".into()), 0, "ts");
|
||||
let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?);
|
||||
let path = cache.object_path(&key).with_extension("ts.tmp.pending");
|
||||
tokio::fs::create_dir_all(path.parent().unwrap()).await?;
|
||||
let staged = cache.adopt_staged_file(path.clone(), 0)?.with_write_budget(Some(budget.clone()))?;
|
||||
let writer = staged.track_writer(tokio::fs::File::create(&path).await?);
|
||||
let mut in_flight_io = writer.lock().await;
|
||||
in_flight_io.as_mut().unwrap().write_all(b"pending").await?;
|
||||
drop(budget);
|
||||
let revision = cache.capacity_revision();
|
||||
drop(staged); // Models cancellation while the producer still owns its I/O handle.
|
||||
let released = cache.wait_for_capacity_change(&revision);
|
||||
tokio::pin!(released);
|
||||
assert!(futures::poll!(released.as_mut()).is_pending());
|
||||
assert!(cache.reserve_revision_peak(&key, 1).await.is_err());
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20);
|
||||
drop(in_flight_io);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), released).await.unwrap();
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
|
||||
assert!(!tokio::fs::try_exists(path).await?);
|
||||
assert!(!cache.has_active_temp_files());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_revision_link_keeps_its_peak_until_the_owned_link_and_unlink_finish() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
cache.update_cache_limits(20, 20);
|
||||
let key = SegmentCacheKey::new(ProxySessionId("pending-link".into()), 0, "ts");
|
||||
let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?);
|
||||
let staged = cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
b"0123456789".as_slice(),
|
||||
Instant::now() + std::time::Duration::from_secs(5),
|
||||
Some(budget.clone()),
|
||||
)
|
||||
.await?;
|
||||
cache.commit_staged(&key, staged).await?;
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
|
||||
let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink);
|
||||
let source = cache.object_path(&key);
|
||||
let publish = cache.publish_processed_revision_with_budget(
|
||||
&revision.revision().key,
|
||||
&source,
|
||||
Instant::now() + std::time::Duration::from_secs(5),
|
||||
Some(budget),
|
||||
);
|
||||
let mut publish = Box::pin(publish);
|
||||
tokio::select! {
|
||||
entered = entered => entered.unwrap(),
|
||||
result = &mut publish => panic!("publication bypassed its link checkpoint: {result:?}"),
|
||||
}
|
||||
drop(publish);
|
||||
assert!(cache.reserve_revision_peak(&key, 1).await.is_err());
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10);
|
||||
let token = cache.capacity_revision();
|
||||
resume.send(()).unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), cache.wait_for_capacity_change(&token)).await.unwrap();
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 10);
|
||||
assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?);
|
||||
assert!(!cache.has_active_temp_files());
|
||||
assert!(cache.metadata(&revision.revision().key).await?.is_none());
|
||||
assert_eq!(tokio::fs::read(source).await?, b"0123456789");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_cache_replacement_cannot_change_a_processed_publication_source_or_its_accounted_size(
|
||||
) -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
cache.update_cache_limits(64, 64);
|
||||
let key = SegmentCacheKey::new(ProxySessionId("publication-source".into()), 0, "ts");
|
||||
cache.write_bytes_and_commit(&key, b"old").await?;
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
|
||||
let source = cache.object_path(&key);
|
||||
let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink);
|
||||
let publication = cache.publish_processed_revision(
|
||||
&revision.revision().key,
|
||||
&source,
|
||||
Instant::now() + std::time::Duration::from_secs(5),
|
||||
);
|
||||
tokio::pin!(publication);
|
||||
tokio::select! {
|
||||
reached = entered => reached.unwrap(),
|
||||
result = &mut publication => panic!("publication bypassed the source checkpoint: {result:?}"),
|
||||
}
|
||||
let replacement = cache.write_bytes_and_commit(&key, b"larger-new-cache-body");
|
||||
tokio::pin!(replacement);
|
||||
assert!(futures::poll!(replacement.as_mut()).is_pending());
|
||||
resume.send(()).unwrap();
|
||||
let (published, replaced) =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async { tokio::join!(publication, replacement) })
|
||||
.await
|
||||
.expect("source lease releases before publication commit reacquires admission");
|
||||
let published = published?;
|
||||
let replaced = replaced?;
|
||||
assert_eq!(published.size, 3);
|
||||
assert_eq!(tokio::fs::read(published.path).await?, b"old");
|
||||
assert_eq!(tokio::fs::read(replaced.path).await?, b"larger-new-cache-body");
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, replaced.size + 3);
|
||||
assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?);
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shrinking_replacement_keeps_old_bytes_charged_until_rename_finishes() -> io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
cache.update_cache_limits(110, 110);
|
||||
let key = SegmentCacheKey::new(ProxySessionId("shrinking".into()), 0, "ts");
|
||||
let other = SegmentCacheKey::new(ProxySessionId("other-session".into()), 0, "ts");
|
||||
cache.write_bytes_and_commit(&key, &[1; 100]).await?;
|
||||
let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeCacheRename);
|
||||
let replacement = cache.write_bytes_and_commit(&key, &[2; 10]);
|
||||
tokio::pin!(replacement);
|
||||
tokio::select! {
|
||||
reached = entered => reached.unwrap(),
|
||||
result = &mut replacement => panic!("replacement bypassed the rename checkpoint: {result:?}"),
|
||||
}
|
||||
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [1; 100]);
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 100);
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10);
|
||||
assert!(cache.reserve_revision_peak(&key, 1).await.is_err());
|
||||
assert!(cache.reserve_revision_peak(&other, 1).await.is_err());
|
||||
resume.send(()).unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), replacement).await.unwrap()?;
|
||||
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [2; 10]);
|
||||
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 10);
|
||||
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
|
||||
let released = cache.reserve_revision_peak(&other, 100).await?;
|
||||
drop(released);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Vendored
+123
-18
@@ -27,9 +27,12 @@ impl HlsRevisionDiskReservation {
|
||||
impl Drop for HlsRevisionDiskReservation {
|
||||
fn drop(&mut self) {
|
||||
let mut capacity = self.capacity.lock_unpoisoned();
|
||||
capacity.revision_reservations.remove(&self.id);
|
||||
let released = capacity.revision_reservations.remove(&self.id).is_some();
|
||||
capacity.reserved_spools.retain(|_, owner| *owner != self.id);
|
||||
if self.unaccounted_writes {
|
||||
capacity.initialized = false;
|
||||
}
|
||||
if released || self.unaccounted_writes {
|
||||
capacity.revision = Arc::new(CapacityRevision);
|
||||
}
|
||||
drop(capacity);
|
||||
@@ -48,6 +51,14 @@ impl HlsSegmentCache {
|
||||
bytes: u64,
|
||||
) -> io::Result<HlsRevisionDiskReservation> {
|
||||
let _admission = self.capacity_admission_gate.lock().await;
|
||||
self.reserve_revision_peak_inner(key, bytes).await
|
||||
}
|
||||
|
||||
pub(super) async fn reserve_revision_peak_inner<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
bytes: u64,
|
||||
) -> io::Result<HlsRevisionDiskReservation> {
|
||||
let root = self.cache_path_snapshot();
|
||||
self.ensure_capacity_initialized(&root.path).await?;
|
||||
let session = key.session_path_component();
|
||||
@@ -77,19 +88,20 @@ impl HlsSegmentCache {
|
||||
HlsCacheCapacityRevision(Arc::clone(&capacity.revision)),
|
||||
));
|
||||
}
|
||||
let id = loop {
|
||||
let id = fastrand::u64(..);
|
||||
if !capacity.revision_reservations.contains_key(&id) {
|
||||
break id;
|
||||
}
|
||||
};
|
||||
let id =
|
||||
capacity.next_write_id.checked_add(1).ok_or_else(|| io::Error::other("cache write identity exhausted"))?;
|
||||
capacity.next_write_id = id;
|
||||
capacity.revision_reservations.insert(id, (session, bytes));
|
||||
capacity.revision = Arc::new(CapacityRevision);
|
||||
drop(capacity);
|
||||
self.capacity_changed.notify_waiters();
|
||||
Ok(HlsRevisionDiskReservation {
|
||||
id,
|
||||
capacity: Arc::clone(&self.capacity),
|
||||
changed: Arc::clone(&self.capacity_changed),
|
||||
unaccounted_writes: false,
|
||||
generation: root.generation,
|
||||
cache_path: root.path,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -101,18 +113,44 @@ impl HlsSegmentCache {
|
||||
}
|
||||
|
||||
pub async fn create_revision_spool<K: HlsCacheObjectKey>(&self, key: &K) -> io::Result<(File, HlsRevisionFilePin)> {
|
||||
self.create_revision_spool_with_budget(key, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn create_revision_spool_with_budget<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
budget: Option<&HlsRevisionDiskReservation>,
|
||||
) -> io::Result<(File, HlsRevisionFilePin)> {
|
||||
let commit_gate = Arc::clone(&self.cache_path_commit_gate).read_owned().await;
|
||||
let root = self.cache_path_snapshot();
|
||||
if budget.is_some_and(|budget| !Arc::ptr_eq(&budget.generation, &root.generation)) {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "revision reservation root retired"));
|
||||
}
|
||||
self.ensure_cache_root_marker_for(&root.path).await?;
|
||||
let path = Self::path_for_key_in(&root.path, key);
|
||||
let pin = self.pin_revision_file(&path)?;
|
||||
let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?;
|
||||
fs::create_dir_all(parent).await?;
|
||||
let file = OpenOptions::new().write(true).create_new(true).open(&path).await?;
|
||||
Ok((file, pin))
|
||||
let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?.to_path_buf();
|
||||
let mut mutation = self.begin_capacity_mutation(&root.path, &path, key.session_path_component()).await;
|
||||
if let Some(budget) = budget {
|
||||
self.register_reserved_spool(path.clone(), budget);
|
||||
}
|
||||
super::staging::run_owned_cache_operation(
|
||||
Arc::clone(&self.owned_operation_permits),
|
||||
"revision-create",
|
||||
async move {
|
||||
fs::create_dir_all(parent).await?;
|
||||
let file = OpenOptions::new().write(true).create_new(true).open(&path).await?;
|
||||
mutation.finish_mutation(None);
|
||||
drop(commit_gate);
|
||||
Ok((file, pin))
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete_orphan_revision_files(&self, cutoff: SystemTime) -> io::Result<usize> {
|
||||
let root = self.cache_path_snapshot();
|
||||
self.ensure_capacity_initialized(&root.path).await?;
|
||||
let Some(mut sessions) = skip_not_found(fs::read_dir(&root.path).await)? else { return Ok(0) };
|
||||
let mut deleted = 0usize;
|
||||
while let Some(session) = sessions.next_entry().await? {
|
||||
@@ -154,23 +192,90 @@ impl HlsSegmentCache {
|
||||
key: &K,
|
||||
source: &Path,
|
||||
deadline: Instant,
|
||||
) -> io::Result<CachedSegmentMetadata> {
|
||||
self.publish_processed_revision_with_budget(key, source, deadline, None).await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines, reason = "Keeps source ownership, peak admission and immutable linking together.")]
|
||||
pub(crate) async fn publish_processed_revision_with_budget<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
source: &Path,
|
||||
deadline: Instant,
|
||||
budget: Option<Arc<HlsRevisionDiskReservation>>,
|
||||
) -> io::Result<CachedSegmentMetadata> {
|
||||
let root = self.cache_path_snapshot();
|
||||
let staged_path = Self::temp_path_for_key(key, &root.path);
|
||||
let size = fs::metadata(source).await?.len();
|
||||
let checks_source_owner = budget.is_some();
|
||||
let budget = match budget {
|
||||
Some(budget) => budget,
|
||||
None => Arc::new(self.reserve_revision_peak_reclaiming(key, size).await?),
|
||||
};
|
||||
let (size, source_lease) = loop {
|
||||
if Instant::now() >= deadline {
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "revision publication timed out"));
|
||||
}
|
||||
self.ensure_write_budget_available(key, &budget, size).await?;
|
||||
// Preserve admission -> path-mutation ordering. A replacement must
|
||||
// not change the source between its size check and the hardlink.
|
||||
let admission = self.capacity_admission_gate.lock().await;
|
||||
let source_lease = self.begin_capacity_mutation(&root.path, source, key.session_path_component()).await;
|
||||
let actual_size = fs::metadata(source).await?.len();
|
||||
let remaining = {
|
||||
let capacity = self.capacity.lock_unpoisoned();
|
||||
if checks_source_owner && capacity.write_owners.get(source) != Some(&budget.id) {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "revision source write replaced"));
|
||||
}
|
||||
capacity.revision_reservations.get(&budget.id).map_or(0, |(_, bytes)| *bytes)
|
||||
};
|
||||
drop(admission);
|
||||
if actual_size <= remaining {
|
||||
break (actual_size, source_lease);
|
||||
}
|
||||
drop(source_lease);
|
||||
self.ensure_write_budget_available(key, &budget, actual_size).await?;
|
||||
};
|
||||
let budget = Some(budget);
|
||||
let registration = self.register_active_temp_path(&staged_path)?;
|
||||
if fs::hard_link(source, &staged_path).await.is_ok() {
|
||||
let size = fs::metadata(&staged_path).await?.len();
|
||||
let staged = StagedCacheObject::registered(staged_path, size, Arc::clone(&root.generation), registration);
|
||||
let staged =
|
||||
StagedCacheObject::registered(staged_path.clone(), size, Arc::clone(&root.generation), registration)
|
||||
.with_write_budget(budget.clone())?;
|
||||
let source_path = source.to_path_buf();
|
||||
#[cfg(test)]
|
||||
let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeRevisionLink);
|
||||
// A Tokio filesystem call can finish after its caller is cancelled.
|
||||
// Retain staging and its budget until the link has a definite result.
|
||||
let (staged, fallback_source) = super::staging::run_owned_cache_operation(
|
||||
Arc::clone(&self.owned_operation_permits),
|
||||
"revision-link",
|
||||
async move {
|
||||
#[cfg(test)]
|
||||
if let Some(checkpoint) = checkpoint {
|
||||
checkpoint.wait().await?;
|
||||
}
|
||||
let fallback = if fs::hard_link(&source_path, &staged_path).await.is_ok() {
|
||||
None
|
||||
} else {
|
||||
// Opening under the same source lease also freezes the
|
||||
// inode used by a copy fallback after replacement resumes.
|
||||
Some(File::open(source_path).await?)
|
||||
};
|
||||
drop(source_lease);
|
||||
Ok((staged, fallback))
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if let Some(file) = fallback_source {
|
||||
self.remove_staged(staged).await?;
|
||||
let staged = self.stage_temp_with_budget(key, file, deadline, budget).await?;
|
||||
self.commit_staged(key, staged).await
|
||||
} else {
|
||||
drop(registration);
|
||||
let file = File::open(source).await?;
|
||||
let staged = self.stage_temp_with_deadline(key, file, deadline).await?;
|
||||
self.commit_staged(key, staged).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Revision file bytes as of the last completed capacity scan.
|
||||
/// Current revision file bytes after completed accounting mutations.
|
||||
pub fn revision_disk_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().revision_bytes }
|
||||
|
||||
/// Walks the cache tree for the current revision file bytes.
|
||||
|
||||
Vendored
+100
-14
@@ -34,7 +34,20 @@ impl StagedCacheObject {
|
||||
cache_path_generation: Arc<CachePathGeneration>,
|
||||
registration: Arc<ActiveTempFileRegistration>,
|
||||
) -> Self {
|
||||
Self { path, size, cache_path_generation, _registration: registration }
|
||||
Self {
|
||||
cleanup: Arc::new(super::StagedFileCleanup {
|
||||
path: path.clone(),
|
||||
registration: Arc::clone(®istration),
|
||||
budget: None,
|
||||
completed: std::sync::atomic::AtomicBool::new(false),
|
||||
writer: std::sync::Mutex::new(None),
|
||||
}),
|
||||
path,
|
||||
size,
|
||||
cache_path_generation,
|
||||
_registration: registration,
|
||||
write_budget: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,9 +131,10 @@ impl HlsSegmentCache {
|
||||
G: Send + 'static,
|
||||
{
|
||||
let staged_path = staged.path.clone();
|
||||
let cleanup = Arc::clone(&staged.cleanup);
|
||||
let result = self.commit_staged_inner(key, staged, guard).await;
|
||||
if result.is_err() {
|
||||
remove_temp_file_after_error(&staged_path, "commit").await;
|
||||
cleanup.completed.store(remove_temp_file_after_error(&staged_path, "commit").await, Ordering::Release);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -164,13 +178,14 @@ impl HlsSegmentCache {
|
||||
};
|
||||
fs::create_dir_all(parent).await?;
|
||||
let reservation = self
|
||||
.prepare_capacity_replacement(
|
||||
.prepare_capacity_replacement_with_budget(
|
||||
&cache_path.path,
|
||||
&final_path,
|
||||
key.session_path_component(),
|
||||
staged.size,
|
||||
self.max_cache_bytes.load(Ordering::Acquire),
|
||||
self.max_session_bytes.load(Ordering::Acquire),
|
||||
staged.write_budget.clone(),
|
||||
)
|
||||
.await;
|
||||
let reservation = match reservation {
|
||||
@@ -265,18 +280,25 @@ impl HlsSegmentCache {
|
||||
drop(admission_gate);
|
||||
let staged_path = staged.path.clone();
|
||||
let committed_size = staged.size;
|
||||
#[cfg(test)]
|
||||
let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeCacheRename);
|
||||
// Once spawned, the owned mutation deliberately outlives cancellation of the requesting future. This
|
||||
// keeps the path lease, temp registration, and accounting reservation alive until the atomic filesystem
|
||||
// operation has a definite result. A runtime shutdown may abort the task, but no cache work can follow
|
||||
// that shutdown; panic/abort drops the reservation and forces accounting rollback/invalidation.
|
||||
return run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "commit", async move {
|
||||
let mut reservation = reservation;
|
||||
#[cfg(test)]
|
||||
if let Some(checkpoint) = checkpoint {
|
||||
checkpoint.wait().await?;
|
||||
}
|
||||
reservation.mark_filesystem_mutation_started();
|
||||
if let Err(err) = fs::rename(&staged_path, &final_path).await {
|
||||
remove_temp_file_after_error(&staged_path, "rename").await;
|
||||
return Err(err);
|
||||
}
|
||||
reservation.finish_replacement();
|
||||
staged.cleanup.completed.store(true, Ordering::Release);
|
||||
drop(staged);
|
||||
drop(commit_gate);
|
||||
Ok((CachedSegmentMetadata { path: final_path, size: committed_size }, guard))
|
||||
@@ -286,11 +308,17 @@ impl HlsSegmentCache {
|
||||
}
|
||||
|
||||
pub async fn remove_staged(&self, staged: StagedCacheObject) -> io::Result<()> {
|
||||
staged.finish_writer().await;
|
||||
#[cfg(test)]
|
||||
self.check_cache_delete_test_failure(&staged.path)?;
|
||||
let result = match fs::remove_file(&staged.path).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if result.is_ok() {
|
||||
staged.cleanup.completed.store(true, Ordering::Release);
|
||||
}
|
||||
drop(staged);
|
||||
result
|
||||
}
|
||||
@@ -315,6 +343,20 @@ impl HlsSegmentCache {
|
||||
reader: &mut R,
|
||||
deadline: Option<Instant>,
|
||||
) -> io::Result<StagedCacheObject>
|
||||
where
|
||||
K: HlsCacheObjectKey,
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
self.stage_temp_inner_with_budget(key, reader, deadline, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn stage_temp_inner_with_budget<K, R>(
|
||||
&self,
|
||||
key: &K,
|
||||
reader: &mut R,
|
||||
deadline: Option<Instant>,
|
||||
budget: Option<Arc<super::HlsRevisionDiskReservation>>,
|
||||
) -> io::Result<StagedCacheObject>
|
||||
where
|
||||
K: HlsCacheObjectKey,
|
||||
R: AsyncRead + Unpin,
|
||||
@@ -322,6 +364,10 @@ impl HlsSegmentCache {
|
||||
if deadline.is_some_and(|deadline| deadline <= Instant::now()) {
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out"));
|
||||
}
|
||||
let budget = match budget {
|
||||
Some(budget) => budget,
|
||||
None => Arc::new(self.reserve_revision_peak_reclaiming(key, 0).await?),
|
||||
};
|
||||
let cache_path = self.cache_path_snapshot();
|
||||
self.ensure_cache_root_marker_for(&cache_path.path).await?;
|
||||
let final_path = Self::path_for_key_in(&cache_path.path, key);
|
||||
@@ -330,28 +376,61 @@ impl HlsSegmentCache {
|
||||
};
|
||||
fs::create_dir_all(parent).await?;
|
||||
|
||||
let (temp_path, mut temp_file, registration) = self.create_temp_file(key, &cache_path).await?;
|
||||
let (temp_path, temp_file, registration) = self.create_temp_file(key, &cache_path).await?;
|
||||
let cleanup = StagedCacheObject::registered(
|
||||
temp_path.clone(),
|
||||
0,
|
||||
Arc::clone(&cache_path.generation),
|
||||
Arc::clone(®istration),
|
||||
)
|
||||
.with_write_budget(Some(Arc::clone(&budget)))?;
|
||||
let writer = cleanup.track_writer(temp_file);
|
||||
if deadline.is_some_and(|deadline| deadline <= Instant::now()) {
|
||||
drop(temp_file);
|
||||
remove_temp_file_after_error(&temp_path, "expired-write").await;
|
||||
cleanup.finish_writer().await;
|
||||
cleanup
|
||||
.cleanup
|
||||
.completed
|
||||
.store(remove_temp_file_after_error(&temp_path, "expired-write").await, Ordering::Release);
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out"));
|
||||
}
|
||||
let copy = async {
|
||||
let mut writer = writer.lock().await;
|
||||
let temp_file = writer.as_mut().ok_or_else(|| io::Error::other("staging writer already closed"))?;
|
||||
let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire);
|
||||
let mut limited = reader.take(max_object_bytes.saturating_add(1));
|
||||
let size = tokio::io::copy(&mut limited, &mut temp_file).await?;
|
||||
let mut size = 0u64;
|
||||
let mut buffer = [0u8; 16 * 1024];
|
||||
loop {
|
||||
let read = limited.read(&mut buffer).await?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
let next_size = size.saturating_add(u64::try_from(read).map_err(io::Error::other)?);
|
||||
if next_size > max_object_bytes {
|
||||
return Err(cache_object_limit_error(max_object_bytes));
|
||||
}
|
||||
// Admission precedes every filesystem write, including headerless
|
||||
// bodies and standalone cache users without a startup transaction.
|
||||
self.ensure_write_budget_available(key, &budget, next_size).await?;
|
||||
temp_file.write_all(&buffer[..read]).await?;
|
||||
size = next_size;
|
||||
}
|
||||
if size > max_object_bytes {
|
||||
return Err(cache_object_limit_error(max_object_bytes));
|
||||
}
|
||||
temp_file.flush().await?;
|
||||
drop(temp_file);
|
||||
writer.take();
|
||||
Ok::<u64, io::Error>(size)
|
||||
};
|
||||
let copy_result = if let Some(deadline) = deadline {
|
||||
if let Ok(result) = timeout_at(deadline, copy).await {
|
||||
result
|
||||
} else {
|
||||
remove_temp_file_after_error(&temp_path, "timed-out-write").await;
|
||||
cleanup.finish_writer().await;
|
||||
cleanup
|
||||
.cleanup
|
||||
.completed
|
||||
.store(remove_temp_file_after_error(&temp_path, "timed-out-write").await, Ordering::Release);
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out"));
|
||||
}
|
||||
} else {
|
||||
@@ -359,10 +438,16 @@ impl HlsSegmentCache {
|
||||
};
|
||||
match copy_result {
|
||||
Ok(size) => {
|
||||
Ok(StagedCacheObject::registered(temp_path, size, Arc::clone(&cache_path.generation), registration))
|
||||
let mut staged = cleanup;
|
||||
staged.size = size;
|
||||
Ok(staged)
|
||||
}
|
||||
Err(err) => {
|
||||
remove_temp_file_after_error(&temp_path, "failed-write").await;
|
||||
cleanup.finish_writer().await;
|
||||
cleanup
|
||||
.cleanup
|
||||
.completed
|
||||
.store(remove_temp_file_after_error(&temp_path, "failed-write").await, Ordering::Release);
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
@@ -411,12 +496,13 @@ impl HlsSegmentCache {
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_temp_file_after_error(path: &Path, operation: &'static str) {
|
||||
async fn remove_temp_file_after_error(path: &Path, operation: &'static str) -> bool {
|
||||
match fs::remove_file(path).await {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Ok(()) => true,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => true,
|
||||
Err(error) => {
|
||||
warn!("HLS cache temporary file cleanup failed: operation={operation} error_kind={:?}", error.kind());
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
use super::HlsSegmentCache;
|
||||
use crate::sync_ext::MutexExt;
|
||||
use std::{
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
#[derive(Clone, Copy, Eq, PartialEq)]
|
||||
pub(crate) enum StartupWritePhase {
|
||||
AfterInventorySnapshot,
|
||||
BeforeNormalCommit,
|
||||
AfterNormalCommit,
|
||||
BeforeRevisionLink,
|
||||
BeforeCacheRename,
|
||||
AfterPublication,
|
||||
}
|
||||
|
||||
pub(super) struct StartupWriteCheckpoint {
|
||||
phase: StartupWritePhase,
|
||||
entered: oneshot::Sender<()>,
|
||||
resume: oneshot::Receiver<()>,
|
||||
}
|
||||
|
||||
impl StartupWriteCheckpoint {
|
||||
pub(super) async fn wait(self) -> io::Result<()> {
|
||||
let _ = self.entered.send(());
|
||||
self.resume.await.map_err(|_| io::Error::from_raw_os_error(28))
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsSegmentCache {
|
||||
pub(crate) fn pause_startup_write(&self, phase: StartupWritePhase) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
|
||||
let (entered, reached) = oneshot::channel();
|
||||
let (resume, released) = oneshot::channel();
|
||||
*self.startup_test_checkpoint.lock_unpoisoned() =
|
||||
Some(StartupWriteCheckpoint { phase, entered, resume: released });
|
||||
(reached, resume)
|
||||
}
|
||||
|
||||
pub(super) fn take_startup_write_checkpoint(&self, phase: StartupWritePhase) -> Option<StartupWriteCheckpoint> {
|
||||
let mut hook = self.startup_test_checkpoint.lock_unpoisoned();
|
||||
if hook.as_ref().is_some_and(|hook| hook.phase == phase) {
|
||||
hook.take()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn startup_write_checkpoint(&self, phase: StartupWritePhase) -> io::Result<()> {
|
||||
if let Some(hook) = self.take_startup_write_checkpoint(phase) {
|
||||
hook.wait().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn fail_next_cache_deletes(&self, path: PathBuf, attempts: usize) {
|
||||
self.delete_test_failures.lock_unpoisoned().insert(path, attempts);
|
||||
}
|
||||
|
||||
pub(super) fn check_cache_delete_test_failure(&self, path: &Path) -> io::Result<()> {
|
||||
let mut failures = self.delete_test_failures.lock_unpoisoned();
|
||||
if let Some(remaining) = failures.get_mut(path).filter(|remaining| **remaining > 0) {
|
||||
*remaining -= 1;
|
||||
return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected cache deletion failure"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+8
-18
@@ -42,6 +42,8 @@ async fn abandoned_capacity_reservation_rolls_back_before_filesystem_mutation()
|
||||
session_component: "session".to_string(),
|
||||
replacement: None,
|
||||
filesystem_mutation_started: false,
|
||||
write_budget: None,
|
||||
transferred_bytes: 0,
|
||||
capacity: Arc::clone(&cache.capacity),
|
||||
changed: Arc::clone(&cache.capacity_changed),
|
||||
};
|
||||
@@ -79,6 +81,8 @@ async fn abandoned_started_filesystem_mutation_invalidates_capacity_snapshot() {
|
||||
session_component: "session".to_string(),
|
||||
replacement: None,
|
||||
filesystem_mutation_started: false,
|
||||
write_budget: None,
|
||||
transferred_bytes: 0,
|
||||
capacity: Arc::clone(&cache.capacity),
|
||||
changed: Arc::clone(&cache.capacity_changed),
|
||||
};
|
||||
@@ -131,22 +135,6 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write
|
||||
cache.write_bytes_and_commit(&second_resident, b"0123456789").await.expect("second resident commits");
|
||||
cache.update_cache_limits(25, 25);
|
||||
|
||||
let first_staged = cache
|
||||
.stage_temp_with_deadline(
|
||||
&first_target,
|
||||
&b"abcdefghij"[..],
|
||||
tokio::time::Instant::now() + Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.expect("first target stages");
|
||||
let second_staged = cache
|
||||
.stage_temp_with_deadline(
|
||||
&second_target,
|
||||
&b"klmnopqrst"[..],
|
||||
tokio::time::Instant::now() + Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.expect("second target stages");
|
||||
let (first_reclaimed_tx, first_reclaimed_rx) = oneshot::channel();
|
||||
let (resume_first_tx, resume_first_rx) = oneshot::channel();
|
||||
let reclaimer = Arc::new(PausingCapacityReclaimer {
|
||||
@@ -158,11 +146,13 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write
|
||||
cache.install_capacity_reclaimer(&reclaimer);
|
||||
|
||||
let first_cache = Arc::clone(&cache);
|
||||
let first_task = tokio::spawn(async move { first_cache.commit_staged(&first_target, first_staged).await });
|
||||
let first_task =
|
||||
tokio::spawn(async move { first_cache.write_bytes_and_commit(&first_target, b"abcdefghij").await });
|
||||
first_reclaimed_rx.await.expect("first reclamation pauses after deleting one resident");
|
||||
|
||||
let second_cache = Arc::clone(&cache);
|
||||
let second_task = tokio::spawn(async move { second_cache.commit_staged(&second_target, second_staged).await });
|
||||
let second_task =
|
||||
tokio::spawn(async move { second_cache.write_bytes_and_commit(&second_target, b"klmnopqrst").await });
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
+393
@@ -0,0 +1,393 @@
|
||||
//! Ownership of peak bytes across staging, rename, and immutable publication.
|
||||
use super::{
|
||||
error::capacity_error, revisions::revision_reserved_bytes, ActiveTempFileRegistration, CacheCapacityPressure,
|
||||
CapacityRevision, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsCacheCapacityRevision,
|
||||
HlsCacheObjectKey, HlsRevisionDiskReservation, HlsSegmentCache, StagedCacheObject,
|
||||
};
|
||||
use crate::sync_ext::MutexExt;
|
||||
use std::{
|
||||
io,
|
||||
path::PathBuf,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
};
|
||||
use tokio::{
|
||||
fs,
|
||||
io::{AsyncRead, AsyncWriteExt},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
type TrackedWriter = Arc<tokio::sync::Mutex<Option<fs::File>>>;
|
||||
|
||||
pub(super) struct StagedFileCleanup {
|
||||
pub(super) path: PathBuf,
|
||||
pub(super) registration: Arc<ActiveTempFileRegistration>,
|
||||
pub(super) budget: Option<Arc<HlsRevisionDiskReservation>>,
|
||||
pub(super) completed: std::sync::atomic::AtomicBool,
|
||||
pub(super) writer: std::sync::Mutex<Option<TrackedWriter>>,
|
||||
}
|
||||
|
||||
impl Drop for StagedFileCleanup {
|
||||
fn drop(&mut self) {
|
||||
if self.completed.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let path = self.path.clone();
|
||||
let registration = Arc::clone(&self.registration);
|
||||
let budget = self.budget.clone();
|
||||
let writer = self.writer.lock_unpoisoned().clone();
|
||||
// Keep both the file registration and peak budget until removal completes.
|
||||
// A successful rename leaves no file at this unique temporary path.
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
runtime.spawn(async move {
|
||||
finish_writer(writer).await;
|
||||
if let Err(error) = fs::remove_file(path).await {
|
||||
if error.kind() != io::ErrorKind::NotFound {
|
||||
log::warn!("HLS staging cleanup deferred: error_kind={:?}", error.kind());
|
||||
if let Some(budget) = &budget {
|
||||
let mut capacity = budget.capacity.lock_unpoisoned();
|
||||
capacity.initialized = false;
|
||||
capacity.revision = Arc::new(CapacityRevision);
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(registration);
|
||||
drop(budget);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_writer(writer: Option<TrackedWriter>) {
|
||||
if let Some(writer) = writer {
|
||||
let mut guard = writer.lock().await;
|
||||
if let Some(mut file) = guard.take() {
|
||||
// Tokio filesystem writes may outlive cancellation of write_all.
|
||||
// Flushing the same handle joins that pending write before unlink.
|
||||
let _ = file.flush().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StagedCacheObject {
|
||||
pub(crate) fn track_writer(&self, file: fs::File) -> TrackedWriter {
|
||||
let writer = Arc::new(tokio::sync::Mutex::new(Some(file)));
|
||||
*self.cleanup.writer.lock_unpoisoned() = Some(Arc::clone(&writer));
|
||||
writer
|
||||
}
|
||||
|
||||
pub(super) async fn finish_writer(&self) {
|
||||
let writer = self.cleanup.writer.lock_unpoisoned().clone();
|
||||
finish_writer(writer).await;
|
||||
}
|
||||
|
||||
pub(crate) fn with_write_budget(mut self, budget: Option<Arc<HlsRevisionDiskReservation>>) -> io::Result<Self> {
|
||||
Arc::get_mut(&mut self.cleanup)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "cannot change a shared staged file's budget"))?
|
||||
.budget
|
||||
.clone_from(&budget);
|
||||
self.write_budget = budget;
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsSegmentCache {
|
||||
pub(crate) async fn stage_temp_with_budget<K, R>(
|
||||
&self,
|
||||
key: &K,
|
||||
mut reader: R,
|
||||
deadline: Instant,
|
||||
budget: Option<Arc<HlsRevisionDiskReservation>>,
|
||||
) -> io::Result<StagedCacheObject>
|
||||
where
|
||||
K: HlsCacheObjectKey,
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
self.stage_temp_inner_with_budget(key, &mut reader, Some(deadline), budget).await
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_write_budget_available<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
budget: &HlsRevisionDiskReservation,
|
||||
size: u64,
|
||||
) -> io::Result<()> {
|
||||
if !Arc::ptr_eq(&self.capacity, &budget.capacity)
|
||||
|| !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation)
|
||||
{
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation"));
|
||||
}
|
||||
let remaining = self
|
||||
.capacity
|
||||
.lock_unpoisoned()
|
||||
.revision_reservations
|
||||
.get(&budget.id)
|
||||
.filter(|(owner, _)| owner == &key.session_path_component())
|
||||
.map(|(_, bytes)| *bytes)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Interrupted, "write reservation no longer current"))?;
|
||||
if size > remaining {
|
||||
if let Err(error) = self.grow_revision_peak(key, budget, size - remaining).await {
|
||||
if let Some(capacity) = super::hls_cache_capacity_from_io(&error) {
|
||||
let mut pressure = capacity.pressure();
|
||||
pressure.staged_bytes = size;
|
||||
return Err(capacity_error(
|
||||
pressure,
|
||||
HlsCacheCapacityReclaimOutcome {
|
||||
protected_working_set_bytes: capacity.protected_working_set_bytes,
|
||||
reclaimable_bytes: capacity.reclaimable_bytes,
|
||||
..Default::default()
|
||||
},
|
||||
capacity.revision().clone(),
|
||||
));
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn grow_revision_peak<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
budget: &HlsRevisionDiskReservation,
|
||||
additional: u64,
|
||||
) -> io::Result<()> {
|
||||
let _admission = self.capacity_admission_gate.lock().await;
|
||||
let root = self.cache_path_snapshot();
|
||||
self.ensure_capacity_initialized(&root.path).await?;
|
||||
let error = match self.try_grow_revision_peak(key, budget, additional) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(error) => error,
|
||||
};
|
||||
let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) };
|
||||
let outcome = self
|
||||
.reclaim_capacity(HlsCacheCapacityReclaimRequest {
|
||||
proxy_session_id: key.proxy_session_id().clone(),
|
||||
target_path: self.object_path(key),
|
||||
required_session_bytes: capacity.required_session_bytes(),
|
||||
required_global_bytes: capacity.required_global_bytes(),
|
||||
staged_bytes: additional,
|
||||
})
|
||||
.await?;
|
||||
self.ensure_capacity_initialized(&root.path).await?;
|
||||
self.try_grow_revision_peak(key, budget, additional).map_err(|error| {
|
||||
match super::hls_cache_capacity_from_io(&error) {
|
||||
Some(capacity) => capacity_error(capacity.pressure(), outcome, capacity.revision().clone()),
|
||||
None => error,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn try_grow_revision_peak<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
budget: &HlsRevisionDiskReservation,
|
||||
additional: u64,
|
||||
) -> io::Result<()> {
|
||||
if !Arc::ptr_eq(&self.capacity, &budget.capacity)
|
||||
|| !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation)
|
||||
{
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired cache write reservation"));
|
||||
}
|
||||
let session = key.session_path_component();
|
||||
let mut capacity = self.capacity.lock_unpoisoned();
|
||||
let (global, local) = revision_reserved_bytes(&capacity, &session);
|
||||
let global = global.saturating_add(capacity.total_bytes);
|
||||
let local = local.saturating_add(capacity.session_bytes.get(&session).copied().unwrap_or_default());
|
||||
let global_limit = self.max_cache_bytes.load(Ordering::Acquire);
|
||||
let local_limit = self.max_session_bytes.load(Ordering::Acquire);
|
||||
if global.saturating_add(additional) > global_limit || local.saturating_add(additional) > local_limit {
|
||||
return Err(capacity_error(
|
||||
CacheCapacityPressure {
|
||||
configured_session_bytes: local_limit,
|
||||
configured_global_bytes: global_limit,
|
||||
current_session_bytes: local,
|
||||
current_global_bytes: global,
|
||||
staged_bytes: additional,
|
||||
required_session_bytes: local.saturating_add(additional).saturating_sub(local_limit),
|
||||
required_global_bytes: global.saturating_add(additional).saturating_sub(global_limit),
|
||||
},
|
||||
HlsCacheCapacityReclaimOutcome::default(),
|
||||
HlsCacheCapacityRevision(Arc::clone(&capacity.revision)),
|
||||
));
|
||||
}
|
||||
let (owner, remaining) = capacity
|
||||
.revision_reservations
|
||||
.get_mut(&budget.id)
|
||||
.filter(|(owner, _)| owner == &session)
|
||||
.ok_or_else(|| io::Error::other("write reservation no longer current"))?;
|
||||
let _ = owner;
|
||||
*remaining = remaining.checked_add(additional).ok_or_else(|| io::Error::other("write budget overflow"))?;
|
||||
capacity.revision = Arc::new(CapacityRevision);
|
||||
drop(capacity);
|
||||
self.capacity_changed.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Admission of a complete fresh attempt; unlike replacement admission this
|
||||
/// cannot credit a leftover stable file against the next attempt's peak.
|
||||
pub(crate) async fn ensure_revision_peak_capacity<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
peak: u64,
|
||||
) -> io::Result<()> {
|
||||
let reservation = self.reserve_revision_peak_reclaiming(key, peak).await?;
|
||||
drop(reservation);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve_revision_peak_reclaiming<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
peak: u64,
|
||||
) -> io::Result<HlsRevisionDiskReservation> {
|
||||
// Reserve and reclaim under one admission gate; the reclaimer must never
|
||||
// reenter admission. It only deletes already-owned files.
|
||||
let _admission = self.capacity_admission_gate.lock().await;
|
||||
let mut outcome = HlsCacheCapacityReclaimOutcome::default();
|
||||
for attempt in 0..2 {
|
||||
match self.reserve_revision_peak_inner(key, peak).await {
|
||||
Ok(reservation) => return Ok(reservation),
|
||||
Err(error) => {
|
||||
let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) };
|
||||
if attempt == 1 {
|
||||
return Err(capacity_error(capacity.pressure(), outcome, capacity.revision().clone()));
|
||||
}
|
||||
outcome = self
|
||||
.reclaim_capacity(HlsCacheCapacityReclaimRequest {
|
||||
proxy_session_id: key.proxy_session_id().clone(),
|
||||
target_path: self.object_path(key),
|
||||
required_session_bytes: capacity.required_session_bytes(),
|
||||
required_global_bytes: capacity.required_global_bytes(),
|
||||
staged_bytes: peak,
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
unreachable!("bounded admission loop returns on second attempt")
|
||||
}
|
||||
|
||||
/// Deletes only the stable file installed by this exact peak owner. The
|
||||
/// path mutation ticket serializes this comparison with every later rename.
|
||||
pub(crate) async fn delete_write_owned<K: HlsCacheObjectKey>(
|
||||
&self,
|
||||
key: &K,
|
||||
budget: &HlsRevisionDiskReservation,
|
||||
) -> io::Result<()> {
|
||||
let path = Self::path_for_key_in(&budget.cache_path, key);
|
||||
let mutation = self.begin_capacity_mutation(&budget.cache_path, &path, key.session_path_component()).await;
|
||||
let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&budget.id);
|
||||
if owned {
|
||||
let result = self.delete_with_reservation(path.clone(), mutation).await;
|
||||
if result.is_err() {
|
||||
self.capacity
|
||||
.lock_unpoisoned()
|
||||
.abandoned_writes
|
||||
.insert(path, (budget.id, key.session_path_component()));
|
||||
}
|
||||
result
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn retry_abandoned_write_deletions(&self) {
|
||||
let pending: Vec<_> = self
|
||||
.capacity
|
||||
.lock_unpoisoned()
|
||||
.abandoned_writes
|
||||
.iter()
|
||||
.take(super::MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN)
|
||||
.map(|(path, owner)| (path.clone(), owner.clone()))
|
||||
.collect();
|
||||
let root = self.cache_path_snapshot();
|
||||
for (path, (owner, session)) in pending {
|
||||
if !path.starts_with(&root.path) {
|
||||
self.capacity.lock_unpoisoned().abandoned_writes.remove(&path);
|
||||
continue;
|
||||
}
|
||||
let mutation = self.begin_capacity_mutation(&root.path, &path, session).await;
|
||||
let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&owner);
|
||||
if owned {
|
||||
let _ = self.delete_with_reservation(path, mutation).await;
|
||||
} else {
|
||||
self.capacity.lock_unpoisoned().abandoned_writes.remove(&path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn register_reserved_spool(&self, path: PathBuf, budget: &HlsRevisionDiskReservation) {
|
||||
let mut capacity = self.capacity.lock_unpoisoned();
|
||||
capacity.write_owners.insert(path.clone(), budget.id);
|
||||
capacity.reserved_spools.insert(path, budget.id);
|
||||
}
|
||||
|
||||
pub(crate) fn account_reserved_spool(
|
||||
&self,
|
||||
path: &std::path::Path,
|
||||
size: u64,
|
||||
budget: &HlsRevisionDiskReservation,
|
||||
) -> io::Result<()> {
|
||||
let mut capacity = self.capacity.lock_unpoisoned();
|
||||
let (session, remaining) = capacity
|
||||
.revision_reservations
|
||||
.get_mut(&budget.id)
|
||||
.ok_or_else(|| io::Error::other("spool reservation expired"))?;
|
||||
let session = session.clone();
|
||||
*remaining = remaining.checked_sub(size).ok_or_else(|| io::Error::other("spool exceeds peak budget"))?;
|
||||
capacity.reserved_spools.remove(path);
|
||||
capacity.total_bytes = capacity.total_bytes.saturating_add(size);
|
||||
*capacity.session_bytes.entry(session).or_default() += size;
|
||||
capacity.revision_bytes = capacity.revision_bytes.saturating_add(size);
|
||||
super::capacity::update_revision_metrics(&capacity);
|
||||
capacity.revision = Arc::new(CapacityRevision);
|
||||
drop(capacity);
|
||||
self.capacity_changed.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn install_revision_metrics(&self, metrics: &Arc<crate::HlsCacheMetrics>) {
|
||||
let mut capacity = self.capacity.lock_unpoisoned();
|
||||
capacity.revision_metrics = Some(Arc::downgrade(metrics));
|
||||
super::capacity::update_revision_metrics(&capacity);
|
||||
}
|
||||
|
||||
pub(crate) fn accounted_session_bytes(&self, session: &crate::ProxySessionId) -> u64 {
|
||||
let key = super::SegmentCacheKey::new(session.clone(), 0, "ts");
|
||||
self.capacity.lock_unpoisoned().session_bytes.get(&key.session_path_component()).copied().unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn accounted_global_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().total_bytes }
|
||||
|
||||
pub(crate) async fn session_revision_disk_bytes(&self, session: &crate::ProxySessionId) -> io::Result<u64> {
|
||||
let key = super::SegmentCacheKey::new(session.clone(), 0, "ts");
|
||||
let mut entries = match fs::read_dir(self.cache_path_snapshot().path.join(key.session_path_component())).await {
|
||||
Ok(entries) => entries,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(0),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mut bytes = 0u64;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
if super::revisions::is_revision_cache_path(&entry.path()) && entry.file_type().await?.is_file() {
|
||||
match entry.metadata().await {
|
||||
Ok(metadata) => bytes = bytes.saturating_add(metadata.len()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn remaining_revision_bytes(&self, session: &crate::ProxySessionId) -> u64 {
|
||||
let key = super::SegmentCacheKey::new(session.clone(), 0, "ts");
|
||||
revision_reserved_bytes(&self.capacity.lock_unpoisoned(), &key.session_path_component()).1
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsRevisionDiskReservation {
|
||||
/// Resolves cleanup against this write's original root, even after handoff.
|
||||
pub(crate) fn object_path<K: HlsCacheObjectKey>(&self, key: &K) -> PathBuf {
|
||||
HlsSegmentCache::path_for_key_in(&self.cache_path, key)
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,7 @@ pub struct HlsGarbageCollector {
|
||||
pending_cache_deletions: Arc<StdMutex<CacheDeletionQueueState>>,
|
||||
access_leases: StdRwLock<Option<Weak<RwLock<HlsAccessLeaseStore>>>>,
|
||||
run_once_gate: AsyncMutex<()>,
|
||||
revision_store: StdRwLock<Option<Weak<super::SegmentRevisionStore>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
@@ -112,6 +113,7 @@ mod deletion;
|
||||
mod policy;
|
||||
mod reclaim;
|
||||
mod report;
|
||||
mod revisions;
|
||||
mod selection;
|
||||
pub use collector::{build_rewrite_secret_fingerprint, exec_hls_cache_gc};
|
||||
use deletion::{CacheDeletionQueueGeneration, CacheObjectDeletion};
|
||||
|
||||
@@ -52,6 +52,7 @@ impl HlsGarbageCollector {
|
||||
pending_cache_deletions: Arc::new(StdMutex::new(CacheDeletionQueueState::default())),
|
||||
access_leases: StdRwLock::new(None),
|
||||
run_once_gate: AsyncMutex::new(()),
|
||||
revision_store: StdRwLock::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +103,11 @@ impl HlsGarbageCollector {
|
||||
report.temp_files_deleted = self.cache.delete_temp_files_older_than(cutoff).await?;
|
||||
|
||||
let sessions = self.sessions.list_sessions().await;
|
||||
if let Some(handle) = sessions.first() {
|
||||
let id = handle.read().await.proxy_session_id.clone();
|
||||
self.cache.capacity_usage(&id).await?;
|
||||
}
|
||||
self.cleanup_revisions(&sessions, &mut report).await?;
|
||||
let mut active_session_ids = HashSet::with_capacity(sessions.len());
|
||||
for session in &sessions {
|
||||
active_session_ids.insert(session.read().await.proxy_session_id.clone());
|
||||
@@ -122,6 +128,7 @@ impl HlsGarbageCollector {
|
||||
session_deletions.persist(&mut report);
|
||||
}
|
||||
|
||||
self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await;
|
||||
let mut global_deletions = self.reserve_cache_deletion_batch();
|
||||
self.collect_global_size_deletions(&sessions, &policy, &mut report, &mut global_deletions).await;
|
||||
global_deletions.persist(&mut report);
|
||||
@@ -131,6 +138,7 @@ impl HlsGarbageCollector {
|
||||
self.remove_idle_session_if_still_idle(session, now_ms, &policy, &mut report).await;
|
||||
}
|
||||
|
||||
self.cleanup_revisions(&sessions, &mut report).await?;
|
||||
report.cache_object_deletions_deferred = self.pending_cache_deletion_count();
|
||||
self.record_report_metrics(&report);
|
||||
if report.did_cleanup_or_invalidate() {
|
||||
|
||||
@@ -4,8 +4,8 @@ use super::{
|
||||
selection::{
|
||||
capacity_reclamation_evidence, collect_projected_session_reclamation, duration_expired_head_segment,
|
||||
fifo_head_size_candidate, oldest_global_fifo_head_candidate, oldest_global_transient_object_candidate,
|
||||
remove_map_entry, remove_segment_entry, remove_stale_queue_entries, session_cache_size,
|
||||
take_expired_transient_objects, total_sessions_cache_size, unprotected_unreferenced_map_ids,
|
||||
remove_map_entry, remove_obsolete_deferred_heads, remove_segment_entry, remove_stale_queue_entries,
|
||||
session_cache_size, take_expired_transient_objects, unprotected_unreferenced_map_ids,
|
||||
},
|
||||
CacheDeletionBatch, GarbageCollectionPolicy, GarbageCollectionReport, HlsCacheCapacityReclaimOutcome,
|
||||
HlsCacheCapacityReclaimRequest, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle,
|
||||
@@ -41,6 +41,8 @@ impl HlsGarbageCollector {
|
||||
self.retry_pending_cache_deletions(&mut report, &mut pending_attempt_budget).await;
|
||||
deletion_attempt_budget =
|
||||
deletion_attempt_budget.saturating_sub(pending_attempts_before.saturating_sub(pending_attempt_budget));
|
||||
let sessions = self.sessions.list_sessions().await;
|
||||
self.cleanup_revisions(&sessions, &mut report).await?;
|
||||
let after_pending = self.cache.capacity_usage(&request.proxy_session_id).await?;
|
||||
let required_session_bytes = request
|
||||
.required_session_bytes
|
||||
@@ -73,6 +75,30 @@ impl HlsGarbageCollector {
|
||||
Some(&request.target_path),
|
||||
release_through,
|
||||
);
|
||||
let head = session.segments.first_key_value();
|
||||
let reason = match head {
|
||||
None => "no-candidates",
|
||||
Some((seq, _)) if protected.segment_proxy_seqs.contains(seq) => "protected",
|
||||
Some((_, entry)) => match entry.status {
|
||||
super::SegmentCacheStatus::Ready { .. } => "reclaimable",
|
||||
super::SegmentCacheStatus::CapacityDeferred { .. } if entry.origin_fetch_ref.is_some() => {
|
||||
"deferred-origin-binding-retained"
|
||||
}
|
||||
super::SegmentCacheStatus::CapacityDeferred { .. } => "obsolete-deferred-head",
|
||||
super::SegmentCacheStatus::Discovered => "not-cached",
|
||||
super::SegmentCacheStatus::Queued { .. } => "queued",
|
||||
super::SegmentCacheStatus::Fetching { .. } => "active-fetch",
|
||||
super::SegmentCacheStatus::FailedRetryable { .. }
|
||||
| super::SegmentCacheStatus::FailedPermanent { .. } => "failed-retention",
|
||||
super::SegmentCacheStatus::Expired => "expired-retention",
|
||||
},
|
||||
};
|
||||
log::debug!(
|
||||
"HLS capacity FIFO: proxy_session={} head={:?} status={:?} blocker={reason}",
|
||||
crate::safe_proxy_session_id(&session.proxy_session_id),
|
||||
head.map(|(seq, _)| *seq),
|
||||
head.map(|(_, entry)| &entry.status)
|
||||
);
|
||||
let evidence = capacity_reclamation_evidence(&session, &protected);
|
||||
protected_working_set_bytes = evidence.protected_working_set_bytes;
|
||||
reclaimable_bytes = evidence.reclaimable_bytes;
|
||||
@@ -95,8 +121,8 @@ impl HlsGarbageCollector {
|
||||
let _ = session.render_and_store_manifest(rendered_at_ms);
|
||||
}
|
||||
} else {
|
||||
protected_working_set_bytes = after_pending.session_bytes;
|
||||
session_lock_busy = true;
|
||||
log::debug!("HLS capacity selection: proxy_session={} outcome=lock-conflict protected_working_set_known=false", crate::safe_proxy_session_id(&request.proxy_session_id));
|
||||
}
|
||||
drop(lease_guard);
|
||||
if session_lock_busy {
|
||||
@@ -112,22 +138,15 @@ impl HlsGarbageCollector {
|
||||
}
|
||||
}
|
||||
|
||||
self.cleanup_revisions(&sessions, &mut report).await?;
|
||||
let after_session = self.cache.capacity_usage(&request.proxy_session_id).await?;
|
||||
let remaining_session_bytes = request
|
||||
.required_session_bytes
|
||||
.saturating_sub(before.session_bytes.saturating_sub(after_session.session_bytes));
|
||||
if remaining_session_bytes > 0 {
|
||||
return Ok(HlsCacheCapacityReclaimOutcome {
|
||||
reclaimed_session_bytes: before.session_bytes.saturating_sub(after_session.session_bytes),
|
||||
reclaimed_global_bytes: before.global_bytes.saturating_sub(after_session.global_bytes),
|
||||
protected_working_set_bytes,
|
||||
reclaimable_bytes,
|
||||
});
|
||||
}
|
||||
let required_global_bytes = request
|
||||
.required_global_bytes
|
||||
.saturating_sub(before.global_bytes.saturating_sub(after_session.global_bytes));
|
||||
if required_global_bytes > 0 {
|
||||
if remaining_session_bytes == 0 && required_global_bytes > 0 {
|
||||
let sessions = self.sessions.list_sessions().await;
|
||||
let mut deletions = self.reserve_cache_deletion_batch();
|
||||
self.collect_projected_global_reclamation(
|
||||
@@ -139,9 +158,13 @@ impl HlsGarbageCollector {
|
||||
)
|
||||
.await;
|
||||
deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await;
|
||||
self.cleanup_revisions(&sessions, &mut report).await?;
|
||||
}
|
||||
|
||||
let after = self.cache.capacity_usage(&request.proxy_session_id).await?;
|
||||
if log::log_enabled!(log::Level::Debug) {
|
||||
self.log_capacity_inventory(&request.proxy_session_id, &report).await;
|
||||
}
|
||||
Ok(HlsCacheCapacityReclaimOutcome {
|
||||
reclaimed_session_bytes: before.session_bytes.saturating_sub(after.session_bytes),
|
||||
reclaimed_global_bytes: before.global_bytes.saturating_sub(after.global_bytes),
|
||||
@@ -150,6 +173,25 @@ impl HlsGarbageCollector {
|
||||
})
|
||||
}
|
||||
|
||||
/// Diagnostic I/O cannot change the result of capacity reclamation.
|
||||
pub(super) async fn log_capacity_inventory(&self, id: &crate::ProxySessionId, report: &GarbageCollectionReport) {
|
||||
let revision_bytes = match self.cache.session_revision_disk_bytes(id).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
log::debug!(
|
||||
"HLS capacity inventory unavailable: proxy_session={} error_kind={:?}",
|
||||
crate::safe_proxy_session_id(id),
|
||||
error.kind()
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let reserved_bytes = self.cache.remaining_revision_bytes(id);
|
||||
log::debug!("HLS capacity inventory: proxy_session={} revision_bytes={revision_bytes} remaining_reserved_bytes={reserved_bytes} revision_deletes_planned={} revision_deletes_succeeded={} revision_deletes_failed={} revision_reclaimed_bytes={}",
|
||||
crate::safe_proxy_session_id(id), report.revision_deletions_planned,
|
||||
report.revision_deletions_succeeded, report.revision_deletions_failed, report.revision_bytes_reclaimed);
|
||||
}
|
||||
|
||||
pub(super) fn collect_session_deletions(
|
||||
session: &mut HlsSession,
|
||||
cache: &HlsSegmentCache,
|
||||
@@ -161,6 +203,7 @@ impl HlsGarbageCollector {
|
||||
report.stale_queue_entries_removed =
|
||||
report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session));
|
||||
|
||||
remove_obsolete_deferred_heads(session, cache, None, None, deletions);
|
||||
let transient_before = session.transient.resources.len();
|
||||
let protected = protected_set_for_cache_reclamation(session, cache, None, None);
|
||||
session.transient.prune_expired_except(now_ms, &protected.key_resource_ids);
|
||||
@@ -195,7 +238,8 @@ impl HlsGarbageCollector {
|
||||
}
|
||||
}
|
||||
|
||||
let mut session_size = session_cache_size(session);
|
||||
let mut session_size =
|
||||
cache.accounted_session_bytes(&session.proxy_session_id).max(session_cache_size(session));
|
||||
while session_size > policy.cache_bytes_per_session && deletions.has_capacity() {
|
||||
let protected = protected_set_for_cache_reclamation(session, cache, None, None);
|
||||
let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids)
|
||||
@@ -247,7 +291,7 @@ impl HlsGarbageCollector {
|
||||
report: &mut GarbageCollectionReport,
|
||||
deletions: &mut CacheDeletionBatch,
|
||||
) {
|
||||
let mut total_size = total_sessions_cache_size(sessions).await;
|
||||
let mut total_size = self.cache.accounted_global_bytes();
|
||||
let mut transient_readiness_advanced = HashSet::new();
|
||||
let mut skipped_transient_sessions = HashSet::new();
|
||||
|
||||
|
||||
@@ -2,6 +2,11 @@ use super::ProxySessionId;
|
||||
|
||||
#[derive(Debug, Default, Clone, Eq, PartialEq)]
|
||||
pub struct GarbageCollectionReport {
|
||||
pub revision_deletions_planned: usize,
|
||||
pub revision_deletions_succeeded: usize,
|
||||
pub revision_deletions_failed: usize,
|
||||
pub revision_bytes_reclaimed: u64,
|
||||
pub revision_lock_conflicts: usize,
|
||||
pub secret_cache_invalidated: bool,
|
||||
pub secret_cache_invalidation_deferred: bool,
|
||||
pub temp_files_deleted: usize,
|
||||
@@ -29,7 +34,9 @@ impl GarbageCollectionReport {
|
||||
}
|
||||
|
||||
pub fn did_cleanup_or_invalidate(&self) -> bool {
|
||||
self.secret_cache_invalidated
|
||||
self.revision_deletions_planned > 0
|
||||
|| self.revision_deletions_failed > 0
|
||||
|| self.secret_cache_invalidated
|
||||
|| self.secret_cache_invalidation_deferred
|
||||
|| self.temp_files_deleted > 0
|
||||
|| self.orphan_session_dirs_deleted > 0
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Shared, bounded retirement for regular GC and projected-capacity reclamation.
|
||||
use super::{GarbageCollectionReport, HlsGarbageCollector, HlsSessionHandle};
|
||||
use crate::{sync_ext::MutexExt, SegmentRevision, SegmentRevisionStore};
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
io,
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
struct RetirementBatch {
|
||||
store: Arc<SegmentRevisionStore>,
|
||||
pending: VecDeque<Arc<SegmentRevision>>,
|
||||
}
|
||||
|
||||
impl Drop for RetirementBatch {
|
||||
fn drop(&mut self) {
|
||||
// Cancellation or a failed delete preserves retry ownership, including
|
||||
// all later entries in the batch which have not reached the filesystem.
|
||||
for revision in self.pending.drain(..) {
|
||||
let _ = self.store.requeue_retired(revision);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsGarbageCollector {
|
||||
pub(crate) fn install_revision_store(&self, store: &Arc<SegmentRevisionStore>) {
|
||||
*self.revision_store.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::downgrade(store));
|
||||
}
|
||||
|
||||
/// Caller already owns the GC gate. Never calls admission or reacquires GC.
|
||||
#[allow(
|
||||
clippy::too_many_lines,
|
||||
reason = "One bounded lease/session snapshot followed by cancellation-safe filesystem retirement."
|
||||
)]
|
||||
pub(super) async fn cleanup_revisions(
|
||||
&self,
|
||||
sessions: &[HlsSessionHandle],
|
||||
report: &mut GarbageCollectionReport,
|
||||
) -> io::Result<()> {
|
||||
self.cache.retry_abandoned_write_deletions().await;
|
||||
let before = (
|
||||
report.revision_deletions_planned,
|
||||
report.revision_deletions_succeeded,
|
||||
report.revision_deletions_failed,
|
||||
report.revision_bytes_reclaimed,
|
||||
);
|
||||
let mut stores: Vec<_> = self
|
||||
.revision_store
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.as_ref()
|
||||
.and_then(Weak::upgrade)
|
||||
.into_iter()
|
||||
.collect();
|
||||
let leases = self
|
||||
.access_leases
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.as_ref()
|
||||
.and_then(Weak::upgrade);
|
||||
for handle in sessions {
|
||||
let mut lease_guard = match &leases {
|
||||
Some(leases) => {
|
||||
if let Ok(guard) = leases.try_write() {
|
||||
Some(guard)
|
||||
} else {
|
||||
report.revision_lock_conflicts += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let Ok(mut session) = handle.try_write() else {
|
||||
report.revision_lock_conflicts += 1;
|
||||
continue;
|
||||
};
|
||||
let id = session.proxy_session_id.clone();
|
||||
let floor = session.segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq);
|
||||
let crate::HlsSession { startup, segments, .. } = &mut *session;
|
||||
if let Some(startup) = startup {
|
||||
startup.revisions.retain(|seq, _| segments.contains_key(seq));
|
||||
if !stores.iter().any(|store| Arc::ptr_eq(store, &startup.store)) {
|
||||
stores.push(Arc::clone(&startup.store));
|
||||
}
|
||||
}
|
||||
if let Some(leases) = &mut lease_guard {
|
||||
leases.prune_revision_bindings(&id, floor);
|
||||
}
|
||||
}
|
||||
let mut remaining = super::MAX_CACHE_DELETE_RETRIES_PER_RUN;
|
||||
for store in stores {
|
||||
if remaining == 0 {
|
||||
break;
|
||||
}
|
||||
let pending = store.retire_unpinned_limit(remaining)?;
|
||||
let mut batch = RetirementBatch { store, pending: pending.into() };
|
||||
let attempts = batch.pending.len();
|
||||
for _ in 0..attempts {
|
||||
let Some(revision) = batch.pending.front().cloned() else { break };
|
||||
report.revision_deletions_planned += 1;
|
||||
remaining -= 1;
|
||||
revision.file_pin.lock_unpoisoned().take();
|
||||
let size = self.cache.metadata(&revision.key).await?.map_or(0, |metadata| metadata.size);
|
||||
match self.cache.delete_if_inactive(&revision.key).await {
|
||||
Ok(()) => {
|
||||
batch.pending.pop_front();
|
||||
report.revision_deletions_succeeded += 1;
|
||||
report.revision_bytes_reclaimed = report.revision_bytes_reclaimed.saturating_add(size);
|
||||
}
|
||||
Err(error) => {
|
||||
report.revision_deletions_failed += 1;
|
||||
log::warn!(
|
||||
"HLS revision delete deferred: proxy_session={} proxy_seq={} error_kind={:?}",
|
||||
crate::safe_proxy_session_id(crate::HlsCacheObjectKey::proxy_session_id(&revision.key)),
|
||||
revision.key.proxy_seq,
|
||||
error.kind()
|
||||
);
|
||||
// Put failed entries at the end of this bounded batch;
|
||||
// its Drop requeues them after successful work finishes.
|
||||
batch.pending.rotate_left(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.metrics.record_revision_gc(
|
||||
report.revision_deletions_planned - before.0,
|
||||
report.revision_deletions_succeeded - before.1,
|
||||
report.revision_deletions_failed - before.2,
|
||||
report.revision_bytes_reclaimed - before.3,
|
||||
);
|
||||
log::debug!(
|
||||
"HLS revision GC: planned={} succeeded={} failed={} reclaimed_bytes={} lock_conflicts={}",
|
||||
report.revision_deletions_planned,
|
||||
report.revision_deletions_succeeded,
|
||||
report.revision_deletions_failed,
|
||||
report.revision_bytes_reclaimed,
|
||||
report.revision_lock_conflicts
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -8,15 +8,6 @@ use super::{
|
||||
};
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
pub(super) async fn total_sessions_cache_size(sessions: &[HlsSessionHandle]) -> u64 {
|
||||
let mut total_size = 0_u64;
|
||||
for session in sessions {
|
||||
let session = session.read().await;
|
||||
total_size = total_size.saturating_add(session_cache_size(&session));
|
||||
}
|
||||
total_size
|
||||
}
|
||||
|
||||
pub(super) fn collect_projected_session_reclamation(
|
||||
session: &mut HlsSession,
|
||||
cache: &HlsSegmentCache,
|
||||
@@ -26,6 +17,7 @@ pub(super) fn collect_projected_session_reclamation(
|
||||
max_deletions: usize,
|
||||
deletions: &mut CacheDeletionBatch,
|
||||
) {
|
||||
remove_obsolete_deferred_heads(session, cache, Some(excluded_path), release_through, deletions);
|
||||
let mut planned_bytes = 0_u64;
|
||||
let mut planned_deletions = 0_usize;
|
||||
let mut transient_readiness_changed = false;
|
||||
@@ -335,3 +327,27 @@ pub(super) fn remove_map_entry(session: &mut HlsSession, map_id: ProxyMapId) ->
|
||||
session.origin_map_to_proxy.retain(|_, mapped_map_id| *mapped_map_id != map_id);
|
||||
Some(MapEntryDeletion { key: map.cache_key, content_length })
|
||||
}
|
||||
|
||||
/// Only retire an inactive deferred FIFO head once its origin binding is gone
|
||||
/// and every existing working-set protection has released it.
|
||||
pub(super) fn remove_obsolete_deferred_heads(
|
||||
session: &mut HlsSession,
|
||||
cache: &HlsSegmentCache,
|
||||
excluded: Option<&std::path::Path>,
|
||||
release_through: Option<u64>,
|
||||
deletions: &mut CacheDeletionBatch,
|
||||
) {
|
||||
while deletions.has_capacity() {
|
||||
let Some((&seq, entry)) = session.segments.first_key_value() else { break };
|
||||
let protected = protected_set_for_cache_reclamation(session, cache, excluded, release_through);
|
||||
if entry.origin_fetch_ref.is_some()
|
||||
|| !matches!(entry.status, SegmentCacheStatus::CapacityDeferred { .. })
|
||||
|| protected.segment_proxy_seqs.contains(&seq)
|
||||
{
|
||||
break;
|
||||
}
|
||||
if let Some(key) = remove_segment_entry(session, seq) {
|
||||
deletions.push(CacheObjectDeletion::Segment { key, reason: SegmentCacheDeletionReason::Duration });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,5 +251,6 @@ mod playlist;
|
||||
mod policy;
|
||||
mod publication;
|
||||
mod recovery;
|
||||
mod revisions;
|
||||
mod streaming;
|
||||
mod transport;
|
||||
|
||||
@@ -191,3 +191,104 @@ async fn transient_object_gc_deletes_the_physical_fill_generation_instead_of_the
|
||||
assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("physical metadata read").is_none());
|
||||
assert!(gc.cache.metadata(&lookup_key).await.expect("logical metadata read").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
|
||||
use crate::{
|
||||
media_reserve::{
|
||||
evaluate_lease_reserve, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackCursor,
|
||||
HlsLeaseReserveInput, HlsManifestCommitIdentity, HlsManifestDeliveryMode,
|
||||
},
|
||||
recovery_timing::{HlsLeaseCutoverTiming, HlsRecoveryTriggerBudgetMs},
|
||||
terminal_tail::HlsMediaContainer,
|
||||
};
|
||||
use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome};
|
||||
|
||||
let mut session = HlsSession::new(crate::HlsSessionKey::new(1, "gc-probe"), b"probe", 1_000);
|
||||
let mut body = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:0\n");
|
||||
for seq in 0..9 {
|
||||
write!(body, "#EXTINF:12.0,\n{seq}.ts\n").expect("test manifest writes");
|
||||
}
|
||||
let OriginManifestParseOutcome::Normal(origin) =
|
||||
parse_origin_media_manifest(&body, "http://origin.example/index.m3u8")
|
||||
else {
|
||||
panic!("normal manifest")
|
||||
};
|
||||
session.apply_origin_manifest(&origin).unwrap();
|
||||
for segment in session.segments.values_mut() {
|
||||
segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 };
|
||||
}
|
||||
session.publishable_origin_head_proxy_seq = Some(4);
|
||||
let mut cursor = HlsLeasePlaybackCursor::default();
|
||||
for seq in 0..8 {
|
||||
let token = cursor.record_request_started(seq, 1_000 + seq * 10_000);
|
||||
let token = cursor.bind_request_timing(token, &session.segments[&seq]);
|
||||
let _ = cursor.record_request_completed(token, 1_000 + seq * 10_000);
|
||||
}
|
||||
let manifest = HlsLeaseManifestSnapshot {
|
||||
startup_revisions: None,
|
||||
delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline,
|
||||
source_commit_identity: HlsManifestCommitIdentity::new(1),
|
||||
uri_materialization: None,
|
||||
finalized_transient_manifest_generation: None,
|
||||
snapshot_generation: 1,
|
||||
delivered_at_ms: 71_000,
|
||||
first_proxy_seq: 4,
|
||||
last_proxy_seq: 6,
|
||||
visible_segments: (4..7)
|
||||
.map(|seq| HlsLeaseManifestSegment {
|
||||
proxy_seq: seq,
|
||||
duration_ms: 12_000,
|
||||
uri: format!("{seq}.ts").into(),
|
||||
discontinuity_before: false,
|
||||
map_ref_ready: true,
|
||||
encryption: None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.into(),
|
||||
discontinuity_sequence: 0,
|
||||
target_duration_ms: 12_000,
|
||||
playlist_duration_ms: 36_000,
|
||||
last_visible_media_end_ms: 36_000,
|
||||
active_map: None,
|
||||
active_encryption: None,
|
||||
container: HlsMediaContainer::MpegTs,
|
||||
};
|
||||
let evaluate = |session: &HlsSession| {
|
||||
let timeline = session.ready_timeline_snapshot(cursor.ready_timeline_start_proxy_seq(4), 71_000);
|
||||
let reserve = evaluate_lease_reserve(HlsLeaseReserveInput {
|
||||
manifest: &manifest,
|
||||
cursor: &cursor,
|
||||
ready_timeline: &timeline,
|
||||
now_ms: 71_000,
|
||||
playback_rate_guard_milli: 1_050,
|
||||
recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000),
|
||||
origin_path_degraded: true,
|
||||
recovery_committed: false,
|
||||
});
|
||||
(timeline.units.len(), reserve)
|
||||
};
|
||||
let (before_units, before) = evaluate(&session);
|
||||
assert_eq!(before.guaranteed_reserve_ms, 34_500);
|
||||
assert!(!ProtectedSet::from_session(&session).segment_proxy_seqs.contains(&0));
|
||||
// Run the actual metadata-removal function used by garbage collection.
|
||||
assert!(remove_segment_entry(&mut session, 0).is_some());
|
||||
let (after_units, after) = evaluate(&session);
|
||||
let still_ready = session.ready_timeline_snapshot(4, 71_000);
|
||||
assert_eq!(still_ready.units.len(), 5);
|
||||
assert_eq!(cursor.first_requested_proxy_seq, Some(0));
|
||||
assert_eq!(before_units, 5);
|
||||
assert_eq!(after_units, 5);
|
||||
assert_eq!(after, before);
|
||||
let timing =
|
||||
HlsLeaseCutoverTiming::from_reserve(71_000, after.guaranteed_reserve_ms, after.transition_margin, None);
|
||||
assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 93_500);
|
||||
// Removing the entire historical prefix must keep exactly the same media clock.
|
||||
for seq in 1..4 {
|
||||
assert!(remove_segment_entry(&mut session, seq).is_some());
|
||||
}
|
||||
assert_eq!(evaluate(&session).1, before);
|
||||
// A real hole in the current READY span must still shorten the reserve.
|
||||
assert!(remove_segment_entry(&mut session, 7).is_some());
|
||||
assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
use super::*;
|
||||
use crate::{SegmentRevisionKind, SegmentRevisionStore};
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostic_inventory_errors_do_not_fail_capacity_reclamation() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
let key = crate::SegmentCacheKey::new(id.clone(), 1, "ts");
|
||||
assert_eq!(gc.cache.capacity_usage(&id).await?.global_bytes, 0);
|
||||
tokio::fs::write(gc.cache.object_path(&key).parent().unwrap(), b"not a directory").await?;
|
||||
assert!(gc.cache.session_revision_disk_bytes(&id).await.is_err());
|
||||
gc.log_capacity_inventory(&id, &GarbageCollectionReport::default()).await;
|
||||
let outcome = gc
|
||||
.reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest {
|
||||
proxy_session_id: id.clone(),
|
||||
target_path: gc.cache.object_path(&key),
|
||||
required_session_bytes: 0,
|
||||
required_global_bytes: 0,
|
||||
staged_bytes: 1,
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(outcome, crate::HlsCacheCapacityReclaimOutcome::default());
|
||||
let occupied = crate::SegmentCacheKey::new(ProxySessionId("occupied".into()), 1, "ts");
|
||||
gc.cache.write_bytes_and_commit(&occupied, &[1; 20]).await?;
|
||||
gc.cache.update_cache_limits(30, 30);
|
||||
let error = gc.cache.reserve_revision_peak_reclaiming(&key, 20).await.err().unwrap();
|
||||
assert!(
|
||||
crate::cache::hls_cache_capacity_from_io(&error).is_some(),
|
||||
"diagnostic I/O must preserve the functional capacity error"
|
||||
);
|
||||
assert_eq!(gc.cache.remaining_revision_bytes(&id), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn global_reclamation_retires_revisions_unpinned_by_another_sessions_segment_deletion() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let (gc, old_session) = gc_with_session(&directory).await;
|
||||
let incoming = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "incoming"), b"secret", 1).await;
|
||||
let old_id = old_session.read().await.proxy_session_id.clone();
|
||||
let incoming_id = incoming.read().await.proxy_session_id.clone();
|
||||
let store = Arc::new(SegmentRevisionStore::default());
|
||||
gc.install_revision_store(&store);
|
||||
let revision = store.create(old_id.clone(), 1, SegmentRevisionKind::Processed)?;
|
||||
let revision_key = revision.revision().key.clone();
|
||||
let metadata = gc.cache.write_bytes_and_commit(&revision_key, &[7; 20]).await?;
|
||||
revision.revision().complete(metadata);
|
||||
*revision.revision().file_pin.lock().unwrap() =
|
||||
Some(gc.cache.pin_revision_file(&gc.cache.object_path(&revision_key))?);
|
||||
let old_key = {
|
||||
let mut session = old_session.write().await;
|
||||
apply_six_segment_manifest_for_gc(&mut session);
|
||||
session.publishable_origin_head_proxy_seq = Some(2);
|
||||
let head = session.segments.get_mut(&1).unwrap();
|
||||
head.origin_fetch_ref = None;
|
||||
head.status = SegmentCacheStatus::Ready { content_length: 20, ready_at_ms: 0 };
|
||||
let key = head.cache_key.clone();
|
||||
let config = tuliprox_core::model::HlsStartupConfig::default();
|
||||
session.startup = Some(crate::HlsSessionStartup {
|
||||
config: config.clone(),
|
||||
first_data_timeout_ms: 1_000,
|
||||
worker: std::sync::Weak::new(),
|
||||
access_leases: Arc::new(RwLock::new(HlsAccessLeaseStore::default())),
|
||||
revisions: [(1, revision)].into(),
|
||||
policy_fixed: true,
|
||||
store,
|
||||
budget: crate::ProgressiveBudgetManager::new(config),
|
||||
});
|
||||
key
|
||||
};
|
||||
gc.cache.write_bytes_and_commit(&old_key, &[1; 20]).await?;
|
||||
gc.cache.update_cache_limits(50, 50);
|
||||
assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40);
|
||||
let key = crate::SegmentCacheKey::new(incoming_id.clone(), 1, "ts");
|
||||
let peak = Arc::new(
|
||||
tokio::time::timeout(Duration::from_secs(2), gc.cache.reserve_revision_peak_reclaiming(&key, 40))
|
||||
.await
|
||||
.expect("global reclamation must finish without a periodic GC")?,
|
||||
);
|
||||
assert!(gc.cache.metadata(&old_key).await?.is_none());
|
||||
assert!(gc.cache.metadata(&revision_key).await?.is_none());
|
||||
assert_eq!(gc.cache.capacity_usage(&old_id).await?.session_bytes, 0);
|
||||
assert!(old_session.read().await.startup.as_ref().unwrap().revisions.is_empty());
|
||||
let staged = gc
|
||||
.cache
|
||||
.stage_temp_with_budget(
|
||||
&key,
|
||||
&[9; 40][..],
|
||||
tokio::time::Instant::now() + Duration::from_secs(2),
|
||||
Some(Arc::clone(&peak)),
|
||||
)
|
||||
.await?;
|
||||
gc.cache.commit_staged(&key, staged).await?;
|
||||
drop(peak);
|
||||
assert_eq!(gc.cache.remaining_revision_bytes(&incoming_id), 0);
|
||||
assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40);
|
||||
assert_eq!(tokio::fs::read(gc.cache.object_path(&key)).await?, [9; 40]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn targeted_reclamation_deletes_only_unpinned_revisions_of_the_same_session() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
let store = Arc::new(SegmentRevisionStore::default());
|
||||
gc.install_revision_store(&store);
|
||||
let metrics = Arc::new(crate::HlsCacheMetrics::default());
|
||||
gc.cache.install_revision_metrics(&metrics);
|
||||
let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?;
|
||||
gc.cache.write_bytes_and_commit(&revision.revision().key, &[7; 80]).await?;
|
||||
let reader = revision.clone();
|
||||
drop(revision);
|
||||
gc.cache.update_cache_limits(100, 100);
|
||||
let next = crate::SegmentCacheKey::new(id.clone(), 1, "ts");
|
||||
assert!(gc.cache.reserve_revision_peak_reclaiming(&next, 30).await.is_err());
|
||||
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80);
|
||||
drop(reader);
|
||||
let peak = gc.cache.reserve_revision_peak_reclaiming(&next, 30).await?;
|
||||
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0);
|
||||
assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?);
|
||||
assert_eq!(metrics.snapshot().revision_disk_bytes, 0);
|
||||
drop(peak);
|
||||
assert_eq!(gc.cache.remaining_revision_bytes(&id), 0);
|
||||
assert_eq!(gc.run_once(0).await?.revision_deletions_succeeded, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_revision_deletes_preserve_bytes_and_are_retried_idempotently() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
let store = Arc::new(SegmentRevisionStore::default());
|
||||
gc.install_revision_store(&store);
|
||||
let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?;
|
||||
let key = revision.revision().key.clone();
|
||||
gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?;
|
||||
gc.cache.fail_next_cache_deletes(gc.cache.object_path(&key), 2);
|
||||
drop(revision);
|
||||
let first = gc.run_once(0).await?;
|
||||
assert_eq!(first.revision_deletions_failed, 2);
|
||||
assert_eq!(first.revision_bytes_reclaimed, 0);
|
||||
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80);
|
||||
assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?);
|
||||
let next = gc.run_once(0).await?;
|
||||
assert_eq!(next.revision_deletions_succeeded, 1);
|
||||
assert_eq!(next.revision_bytes_reclaimed, 80);
|
||||
assert!(!tokio::fs::try_exists(gc.cache.object_path(&key)).await?);
|
||||
assert_eq!(gc.run_once(0).await?.revision_deletions_planned, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn obsolete_deferred_fifo_head_retires_without_skipping_a_protected_head() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
let key = {
|
||||
let mut session = session.write().await;
|
||||
apply_six_segment_manifest_for_gc(&mut session);
|
||||
let head = session.segments.get_mut(&1).unwrap();
|
||||
head.status =
|
||||
SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 1 };
|
||||
head.origin_fetch_ref = None;
|
||||
head.access.reader_started(1);
|
||||
head.cache_key.clone()
|
||||
};
|
||||
gc.cache.write_bytes_and_commit(&key, b"partial-commit").await?;
|
||||
let first = gc.run_once(0).await?;
|
||||
assert_eq!(first.segments_deleted(), 0);
|
||||
assert!(session.read().await.segments.contains_key(&1));
|
||||
session.write().await.segments.get_mut(&1).unwrap().access.reader_finished();
|
||||
let second = gc.run_once(0).await?;
|
||||
assert_eq!(second.segments_deleted(), 1);
|
||||
assert!(!session.read().await.segments.contains_key(&1));
|
||||
assert!(gc.cache.metadata(&key).await?.is_none());
|
||||
assert_eq!(gc.run_once(0).await?.segments_deleted(), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn revision_cleanup_skips_contended_locks_and_pressure_keeps_protection_unknown() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
let leases = Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default()));
|
||||
gc.install_access_leases(&leases);
|
||||
let lease_guard = leases.write().await;
|
||||
let mut report = GarbageCollectionReport::default();
|
||||
tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report))
|
||||
.await
|
||||
.expect("revision cleanup must not wait while the lease lock is held")?;
|
||||
assert_eq!(report.revision_lock_conflicts, 1);
|
||||
drop(lease_guard);
|
||||
|
||||
let key = crate::SegmentCacheKey::new(id.clone(), 0, "ts");
|
||||
gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?;
|
||||
let session_guard = session.write().await;
|
||||
let mut report = GarbageCollectionReport::default();
|
||||
tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report))
|
||||
.await
|
||||
.expect("revision cleanup must not reverse the lease/session lock order")?;
|
||||
assert_eq!(report.revision_lock_conflicts, 1);
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
gc.reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest {
|
||||
proxy_session_id: id.clone(),
|
||||
target_path: gc.cache.object_path(&key),
|
||||
required_session_bytes: 1,
|
||||
required_global_bytes: 0,
|
||||
staged_bytes: 1,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("targeted selection must not wait for a contended session")?;
|
||||
assert_eq!(outcome.protected_working_set_bytes, 0, "a lock conflict cannot claim all 80 bytes are protected");
|
||||
assert_eq!(outcome.reclaimed_session_bytes, 0);
|
||||
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80);
|
||||
drop(session_guard);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shared_revision_cleanup_retires_a_bounded_batch_and_drains_idempotently() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
let store = Arc::new(SegmentRevisionStore::default());
|
||||
gc.install_revision_store(&store);
|
||||
let limit = super::super::MAX_CACHE_DELETE_RETRIES_PER_RUN;
|
||||
for seq in 0..limit + 3 {
|
||||
let revision = store.create(id.clone(), u64::try_from(seq).unwrap(), SegmentRevisionKind::Processed)?;
|
||||
gc.cache.write_bytes_and_commit(&revision.revision().key, &[7]).await?;
|
||||
}
|
||||
for expected in [limit, 3, 0] {
|
||||
let mut report = GarbageCollectionReport::default();
|
||||
gc.cleanup_revisions(&[Arc::clone(&session)], &mut report).await?;
|
||||
assert_eq!(report.revision_deletions_planned, expected);
|
||||
assert_eq!(report.revision_deletions_succeeded, expected);
|
||||
assert_eq!(report.revision_deletions_failed, 0);
|
||||
assert_eq!(report.revision_bytes_reclaimed, u64::try_from(expected).unwrap());
|
||||
assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?);
|
||||
}
|
||||
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0);
|
||||
assert_eq!(gc.cache.remaining_revision_bytes(&id), 0);
|
||||
Ok(())
|
||||
}
|
||||
@@ -68,8 +68,12 @@ async fn global_size_gc_subtracts_map_bytes_after_segment_removal() {
|
||||
session.apply_origin_manifest(&manifest).expect("manifest should map");
|
||||
for (proxy_seq, ready_at_ms) in [(0, 0), (1, 1)] {
|
||||
let segment = session.segments.get_mut(&proxy_seq).expect("segment");
|
||||
gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("segment fixture");
|
||||
segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms };
|
||||
}
|
||||
for map in session.maps.values() {
|
||||
gc.cache.write_bytes_and_commit(&map.cache_key, &[0; 10]).await.expect("map fixture");
|
||||
}
|
||||
session.maps.get_mut(&ProxyMapId(0)).expect("first map").status =
|
||||
MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 };
|
||||
session.maps.get_mut(&ProxyMapId(1)).expect("second map").status =
|
||||
|
||||
@@ -268,3 +268,21 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod short_window_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strip_preserves_every_segment_of_an_already_short_playlist() {
|
||||
for count in [1, 2] {
|
||||
for mode in [HlsStripMode::Segments, HlsStripMode::Seconds] {
|
||||
let strip = StripConfig { mode, value: 30 };
|
||||
let body = format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n{}", "#EXTINF:2.5,\nsegment.ts\n".repeat(count));
|
||||
let view = materialize_initial_hls_strip_view(&body, &strip, HlsManifestWindowPolicy::ApplyLiveWindow);
|
||||
assert_eq!(view.body, body);
|
||||
assert_eq!(initial_hls_strip_segments_for_durations(&strip, &vec![2_500; count]), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
//! Finite custom media published after the live splice deadline has elapsed.
|
||||
|
||||
use crate::{
|
||||
hls_ctx::HlsCtx,
|
||||
lease::{HlsAccessLeaseId, HlsTerminalTailPreparation},
|
||||
prepared_terminal_bundle::{prepared_terminal_bundle_key, HlsPreparedTerminalBundle},
|
||||
runtime_custom_tail::{
|
||||
await_prepared_standalone_bundle, snapshot_hls_runtime_custom_tail_asset, standalone_custom_asset_fingerprint,
|
||||
HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, HlsStandaloneCustomPlan,
|
||||
HlsStandaloneCustomPlanKey,
|
||||
},
|
||||
terminal_tail::{HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailGeneration},
|
||||
HlsAccessLeaseState, ProxySessionId,
|
||||
};
|
||||
use std::{fmt, fmt::Write, sync::Arc};
|
||||
use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs};
|
||||
|
||||
/// Immutable, lease-bound custom playlist that does not consume a live splice base.
|
||||
#[derive(Clone)]
|
||||
pub struct HlsLateCustomResponsePlan {
|
||||
pub generation: HlsTerminalTailGeneration,
|
||||
pub(crate) manifest_snapshot_generation: u64,
|
||||
pub(crate) key: HlsStandaloneCustomPlanKey,
|
||||
proxy_session_id: ProxySessionId,
|
||||
lease_id: HlsAccessLeaseId,
|
||||
standalone: HlsStandaloneCustomPlan,
|
||||
}
|
||||
|
||||
impl HlsLateCustomResponsePlan {
|
||||
pub fn matches_route(&self, proxy_session_id: &ProxySessionId, lease_id: &HlsAccessLeaseId) -> bool {
|
||||
self.proxy_session_id == *proxy_session_id && self.lease_id == *lease_id
|
||||
}
|
||||
|
||||
pub fn manifest_body(&self) -> &str { &self.standalone.manifest_body }
|
||||
|
||||
pub(crate) fn segment_bytes(
|
||||
&self,
|
||||
fingerprint: &str,
|
||||
index: u16,
|
||||
) -> Result<bytes::Bytes, crate::runtime_custom_tail::HlsStandaloneCustomSegmentError> {
|
||||
use crate::runtime_custom_tail::HlsStandaloneCustomSegmentError;
|
||||
if fingerprint != standalone_custom_asset_fingerprint(self.key.asset) {
|
||||
return Err(HlsStandaloneCustomSegmentError::StaleAssetFingerprint);
|
||||
}
|
||||
self.standalone.segment_bytes(index).ok_or(HlsStandaloneCustomSegmentError::InvalidIndex)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for HlsLateCustomResponsePlan {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("HlsLateCustomResponsePlan")
|
||||
.field("generation", &self.generation)
|
||||
.field("key", &self.key)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for HlsLateCustomResponsePlan {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.generation == other.generation
|
||||
&& self.manifest_snapshot_generation == other.manifest_snapshot_generation
|
||||
&& self.key == other.key
|
||||
&& self.proxy_session_id == other.proxy_session_id
|
||||
&& self.lease_id == other.lease_id
|
||||
&& self.standalone.manifest_body == other.standalone.manifest_body
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for HlsLateCustomResponsePlan {}
|
||||
|
||||
pub(crate) enum HlsLateCustomResponseError {
|
||||
Superseded,
|
||||
Unavailable(HlsTerminalTailCompatibility),
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_late_custom_response(
|
||||
ctx: &HlsCtx,
|
||||
proxy_session_id: &ProxySessionId,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &HlsTerminalTailPreparation,
|
||||
now_ms: u64,
|
||||
) -> Result<(Arc<HlsLateCustomResponsePlan>, HlsRuntimeCustomTailAssetIdentity), HlsLateCustomResponseError> {
|
||||
let lease = ctx
|
||||
.hls_proxy
|
||||
.access_lease_response_snapshot(lease_id, proxy_session_id, now_ms)
|
||||
.await
|
||||
.ok_or(HlsLateCustomResponseError::Superseded)?;
|
||||
if !matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle)
|
||||
|| lease.playback_mode != HlsLeasePlaybackMode::Live
|
||||
|| lease.issued_at_ms != preparation.lease_issued_at_ms
|
||||
|| lease.admission_generation != preparation.expected_admission_generation
|
||||
|| lease.playback_cursor.cursor_generation != preparation.cursor_generation
|
||||
|| lease.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation)
|
||||
!= Some(preparation.manifest_snapshot_generation)
|
||||
{
|
||||
return Err(HlsLateCustomResponseError::Superseded);
|
||||
}
|
||||
let snapshot = snapshot_hls_runtime_custom_tail_asset(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable)
|
||||
.map_err(|error| HlsLateCustomResponseError::Unavailable(error.terminal_compatibility()))?;
|
||||
let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&snapshot);
|
||||
let bundle_key = prepared_terminal_bundle_key(
|
||||
&snapshot.asset,
|
||||
preparation.manifest_snapshot.target_duration_ms,
|
||||
crate::terminal_tail::HLS_TERMINAL_TAIL_SEGMENT_COUNT,
|
||||
);
|
||||
let bundle = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), bundle_key)
|
||||
.await
|
||||
.map_err(HlsLateCustomResponseError::Unavailable)?;
|
||||
let first_proxy_seq = lease
|
||||
.published_media_sequence_highwater
|
||||
.unwrap_or(preparation.manifest_snapshot.last_proxy_seq)
|
||||
.checked_add(1)
|
||||
.ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?;
|
||||
let discontinuity_sequence =
|
||||
lease.published_discontinuity_sequence_highwater.max(preparation.manifest_snapshot.discontinuity_sequence);
|
||||
let path_prefix = preparation
|
||||
.manifest_snapshot
|
||||
.visible_segments
|
||||
.first()
|
||||
.and_then(|segment| {
|
||||
let uri = preparation.manifest_snapshot.materialize_uri(&segment.uri);
|
||||
uri.find("/hls/shared/live/").map(|offset| uri[..offset].to_owned())
|
||||
})
|
||||
.ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::InvalidLeaseRoute))?;
|
||||
let manifest_body = render_late_manifest(first_proxy_seq, discontinuity_sequence, &path_prefix, lease_id, &bundle)?;
|
||||
let key = HlsStandaloneCustomPlanKey {
|
||||
reason: snapshot.reason,
|
||||
asset: expected_asset.media,
|
||||
segment_count: bundle.key.segment_count,
|
||||
};
|
||||
let standalone = HlsStandaloneCustomPlan { key, manifest_body: Arc::from(manifest_body), prepared_bundle: bundle };
|
||||
Ok((
|
||||
Arc::new(HlsLateCustomResponsePlan {
|
||||
generation: HlsTerminalTailGeneration(preparation.decision_generation),
|
||||
manifest_snapshot_generation: preparation.manifest_snapshot_generation,
|
||||
key,
|
||||
proxy_session_id: proxy_session_id.clone(),
|
||||
lease_id: lease_id.clone(),
|
||||
standalone,
|
||||
}),
|
||||
expected_asset,
|
||||
))
|
||||
}
|
||||
|
||||
// Keeping the layout separate makes the publication boundary testable without a live origin or cache.
|
||||
fn render_late_manifest(
|
||||
first_proxy_seq: u64,
|
||||
discontinuity_sequence: u64,
|
||||
path_prefix: &str,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
bundle: &HlsPreparedTerminalBundle,
|
||||
) -> Result<String, HlsLateCustomResponseError> {
|
||||
first_proxy_seq
|
||||
.checked_add(u64::from(bundle.key.segment_count).saturating_sub(1))
|
||||
.ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?;
|
||||
let fingerprint = standalone_custom_asset_fingerprint(bundle.key.asset);
|
||||
let extinf = format_hls_duration_ms(bundle.source_asset_duration_ms);
|
||||
let mut manifest = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{}\n#EXT-X-MEDIA-SEQUENCE:{first_proxy_seq}\n#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuity_sequence}\n#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n", hls_target_duration_secs(bundle.key.target_duration_ms));
|
||||
for index in 0..bundle.key.segment_count {
|
||||
let _ = writeln!(manifest, "#EXTINF:{extinf},\n{path_prefix}/cvs/hls/{}/{fingerprint}/{index}.ts", lease_id.0);
|
||||
}
|
||||
manifest.push_str("#EXT-X-ENDLIST\n");
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,47 @@
|
||||
use super::*;
|
||||
use crate::{
|
||||
prepared_terminal_bundle::{HlsPreparedTerminalBundleKey, HlsPreparedTerminalSegment},
|
||||
terminal_tail::HlsTerminalAssetIdentity,
|
||||
};
|
||||
|
||||
fn bundle() -> HlsPreparedTerminalBundle {
|
||||
HlsPreparedTerminalBundle {
|
||||
key: HlsPreparedTerminalBundleKey {
|
||||
asset: HlsTerminalAssetIdentity { revision: 7, fingerprint: [7; 32] },
|
||||
target_duration_ms: 12_000,
|
||||
segment_count: 12,
|
||||
},
|
||||
source_asset_duration_ms: 10_027,
|
||||
source_asset_duration_ticks_90khz: 902_400,
|
||||
segments: (0..12)
|
||||
.map(|index| HlsPreparedTerminalSegment {
|
||||
index,
|
||||
timestamp_offset_ticks_90khz: u64::from(index) * 902_400,
|
||||
bytes: bytes::Bytes::from_static(b"prepared terminal segment"),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_manifest_keeps_target_and_sequences_and_contains_only_custom_media() {
|
||||
let bundle = bundle();
|
||||
let manifest = render_late_manifest(131, 9, "/iptv", &HlsAccessLeaseId("lease".into()), &bundle)
|
||||
.unwrap_or_else(|_| panic!("valid late manifest"));
|
||||
assert!(manifest.contains("#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:131\n#EXT-X-DISCONTINUITY-SEQUENCE:9\n"));
|
||||
assert_eq!(manifest.matches("#EXTINF:10.027,").count(), 12);
|
||||
assert_eq!(manifest.matches("#EXT-X-DISCONTINUITY\n").count(), 1);
|
||||
assert!(manifest.contains("#EXT-X-KEY:METHOD=NONE\n"));
|
||||
assert!(manifest.contains("/iptv/cvs/hls/lease/0000000000000007/11.ts"));
|
||||
assert!(!manifest.contains("/hls/shared/live/"));
|
||||
assert!(manifest.ends_with("#EXT-X-ENDLIST\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_manifest_rejects_sequence_overflow_instead_of_reusing_live_sequences() {
|
||||
let bundle = bundle();
|
||||
assert!(matches!(
|
||||
render_late_manifest(u64::MAX - 1, 0, "", &HlsAccessLeaseId("lease".into()), &bundle,),
|
||||
Err(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))
|
||||
));
|
||||
}
|
||||
@@ -144,6 +144,8 @@ pub struct HlsAccessLease {
|
||||
pub startup_admission: HlsLeaseStartupAdmissionState,
|
||||
pub playback_cursor: HlsLeasePlaybackCursor,
|
||||
pub last_manifest_snapshot: Option<HlsLeaseManifestSnapshot>,
|
||||
pub(crate) published_media_sequence_highwater: Option<u64>,
|
||||
pub(crate) published_discontinuity_sequence_highwater: u64,
|
||||
revision_bindings: Option<Arc<HlsLeaseRevisionBindings>>,
|
||||
progressive_startup_claimed: bool,
|
||||
published_transient_resource_ids: HlsPublishedTransientResourceIds,
|
||||
|
||||
@@ -37,7 +37,9 @@ impl HlsAccessLease {
|
||||
HlsLeasePlaybackMode::TerminalTail(plan) => {
|
||||
HlsMediaLeasePlaybackIdentity::TerminalTail { generation: plan.generation }
|
||||
}
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return None,
|
||||
HlsLeasePlaybackMode::TerminalCustomResponse(_)
|
||||
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
|
||||
| HlsLeasePlaybackMode::Ended => return None,
|
||||
};
|
||||
Some(HlsMediaLeaseIdentity { issued_at_ms: self.issued_at_ms, playback })
|
||||
}
|
||||
|
||||
@@ -93,7 +93,9 @@ impl HlsAccessLease {
|
||||
HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => {
|
||||
Some(HlsTerminalTailGeneration(decision_generation))
|
||||
}
|
||||
HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None,
|
||||
HlsLeasePlaybackMode::Live
|
||||
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
|
||||
| HlsLeasePlaybackMode::Ended => None,
|
||||
};
|
||||
if self.pending_terminal_protection_release.is_none() {
|
||||
self.pending_terminal_protection_release = release_generation;
|
||||
@@ -370,6 +372,24 @@ impl HlsAccessLeaseStore {
|
||||
Some(token)
|
||||
}
|
||||
|
||||
pub(crate) fn record_timed_segment_request_started_if_identity_matches(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
proxy_session_id: &ProxySessionId,
|
||||
expected: HlsMediaLeaseIdentity,
|
||||
segment: &crate::SegmentEntry,
|
||||
requested_at_ms: u64,
|
||||
) -> Option<HlsPlaybackRequestToken> {
|
||||
let token = self.record_segment_request_started_if_identity_matches(
|
||||
lease_id,
|
||||
proxy_session_id,
|
||||
expected,
|
||||
segment.proxy_seq,
|
||||
requested_at_ms,
|
||||
)?;
|
||||
Some(self.by_lease_id.get_mut(lease_id)?.playback_cursor.bind_request_timing(token, segment))
|
||||
}
|
||||
|
||||
pub fn record_segment_request_completed_if_identity_matches(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user