From 5ae547f5e5a000380de7fcdcec4233bfe4671363 Mon Sep 17 00:00:00 2001 From: knylbyte <40831653+knylbyte@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:42:50 +0200 Subject: [PATCH 1/2] FIX: stabilize shared HLS playback, recovery and account handovers (#925) * fix (hls): preserve playback reserve across cache eviction keep a stable session media clock and completed segment timings so cache eviction and origin rollovers cannot reset playback reserve. update segment fixtures and cover reserve stability after collection. * fix (hls): prevent playback stalls during origin manifest retries mark the origin as degraded on the first retryable manifest failure, cap retry-after delays, and reevaluate reserve independently of client polling. use a bounded state commit after an expired splice deadline instead of leaving the lease live when terminal media cannot be published safely. * fix (hls): serve custom responses after the live splice deadline publish a finite lease-bound channel_unavailable playlist on the existing manifest route when a safe live splice is no longer possible. renew lease and session activity on successful custom-segment requests and retain concrete preparation failure diagnostics. * fix(hls): persist origin qualification for short playlists Share qualification and retry receipts across leases, preserve them while media publication is pending, and allow controlled recovery after expired acceptance episodes. Prefer eligible regular candidates before short fallbacks. Add startup, revalidation, replay, cookie and endpoint regression coverage, and document short-window behavior. Validation during implementation: Linux make test (7009 passed, 23 ignored), focused HLS and endpoint suites, format checks, Clippy and Markdown lint. Offline production library and backend checks passed after the import fix. * fix(hls): reconcile cache write budgets and revision reclamation Transfer reserved bytes atomically through staging and publication, preserve cleanup ownership across cancellation, and retire unpinned revisions during local and global reclamation. Version temp registrations during inventory scans, remove production panic calls, and isolate debug diagnostics from admission results. Validation: 1,162 HLS tests and 7,038 workspace tests passed; fmt-check, lint, test, markdown-lint and isolated production Clippy passed with dev-slim. * feat (ai): add local codex files * fix (hls): replace invalid origin accounts in shared HLS sessions * fix (hls): synchronize origin account handovers across WebUI viewers * fix (hls): preserve same-account resume work and backpressure * fix (hls): resume reduced retries without crossing generations Allow eligible exhausted acceptance episodes to select reduced retry candidates while preserving generation, candidate, workload and timing guards. Keep recovery chains bound to their generation, including changes between selection and commit. Add regressions for repaired and persistent resource conflicts, generation isolation and media delivery through the app's local HTTP origin. Keep shared manifest fixtures under test/fixtures/hls/reduced-retry. --- .gitignore | 2 + Cargo.lock | 1 + Cargo.toml | 8 - backend/app/Cargo.toml | 1 + .../tests/admission_eviction_policy.rs | 10 +- .../stream_responses_provider_fallback.rs | 4 +- backend/app/src/api/config_file.rs | 4 +- .../api/endpoints/custom_video_stream_api.rs | 27 +- .../endpoints/hls_api/manifest/admission.rs | 62 +- .../hls_api/manifest/canonical_owner.rs | 2 +- .../hls_api/manifest/provisioning.rs | 15 +- backend/app/src/api/endpoints/hls_api/mod.rs | 4 +- .../hls_api/origin_account_runtime.rs | 719 +++++++ .../hls_api/origin_refresh_request.rs | 58 + .../api/endpoints/hls_api/segment/access.rs | 4 +- .../api/endpoints/hls_api/segment/fallback.rs | 8 +- .../endpoints/hls_api/segment/registration.rs | 4 + .../endpoints/hls_api/segment/resources.rs | 21 +- .../api/endpoints/hls_api/segment/segments.rs | 8 +- .../hls_api/segment/transient_cache.rs | 44 +- .../app/src/api/endpoints/hls_api/session.rs | 609 ++---- .../app/src/api/endpoints/hls_api/tests.rs | 5 + .../endpoints/hls_api/tests/entry_identity.rs | 10 +- .../hls_api/tests/late_custom_response.rs | 259 +++ .../hls_api/tests/manifest_entry_freshness.rs | 29 +- .../hls_api/tests/manifest_short_window.rs | 526 +++++ .../hls_api/tests/manifest_startup.rs | 2 +- .../hls_api/tests/origin_account_rebind.rs | 1725 +++++++++++++++++ .../endpoints/hls_api/tests/owner_token.rs | 4 +- .../hls_api/tests/provider_admission.rs | 3 + .../provider_admission_account_overlap.rs | 93 +- .../provider_admission_provider_binding.rs | 101 +- ...ovider_admission_provider_lease_release.rs | 2 +- .../endpoints/hls_api/tests/reduced_retry.rs | 287 +++ .../hls_api/tests/same_account_resume.rs | 294 +++ .../hls_api/tests/support/app_state.rs | 1 + .../hls_api/tests/support/manifest.rs | 2 + .../hls_api/tests/support/runtime_policy.rs | 1 + .../hls_api/tests/support/sessions.rs | 2 +- .../endpoints/hls_api/tests/terminal_tail.rs | 32 +- .../tests/terminal_tail_terminal_cutover.rs | 26 +- .../api/endpoints/hls_terminal_response.rs | 8 + .../app/src/api/endpoints/v1_api_config.rs | 24 +- .../tests/behavior_manual_update.rs | 6 +- .../v1_api_playlist/tests/persistence.rs | 2 +- .../app/src/api/endpoints/xtream_api/tests.rs | 2 +- .../app/tests/hls_origin_account_handover.rs | 353 ++++ .../core/src/utils/network/content_coding.rs | 4 +- backend/core/src/utils/network/request.rs | 4 +- .../utils/network/request/tests/behavior.rs | 18 +- .../network/request/tests/persistence.rs | 2 +- backend/curation/src/coordinator/tests.rs | 2 +- backend/curation/src/kernel.rs | 4 +- backend/curation/src/test_support.rs | 2 +- backend/curation/src/tmdb/client/tests.rs | 6 +- backend/curation/src/trakt/mod.rs | 2 +- .../dvr/src/recording/recording_conflict.rs | 2 +- .../dvr/src/recording/recording_deletion.rs | 2 +- .../recording/recording_queue/tests/policy.rs | 2 +- .../dvr/src/recording/recording_retention.rs | 4 +- .../src/recording/recording_rule_scheduler.rs | 14 +- backend/dvr/src/recording/recording_worker.rs | 15 +- backend/dvr/src/recording/recording_ws.rs | 4 +- .../hls/src/availability/manifest_failure.rs | 46 + backend/hls/src/availability/mod.rs | 9 +- backend/hls/src/availability/reevaluation.rs | 34 +- .../hls/src/availability/terminal_cutover.rs | 72 +- .../availability/terminal_pending_owner.rs | 78 +- backend/hls/src/availability/tests.rs | 1 + .../tests/late_custom_response.rs | 532 +++++ .../src/availability/tests/owner_fixtures.rs | 1 + .../hls/src/availability/tests/recovery.rs | 284 +++ .../availability/tests/runtime_fixtures.rs | 3 +- .../availability/tests/terminal_fixtures.rs | 6 +- backend/hls/src/availability_reevaluation.rs | 43 +- backend/hls/src/cache.rs | 32 +- backend/hls/src/cache/capacity.rs | 190 +- backend/hls/src/cache/lifecycle.rs | 42 +- .../revision_capacity_tests/publication.rs | 448 +++++ backend/hls/src/cache/revisions.rs | 141 +- backend/hls/src/cache/staging.rs | 114 +- backend/hls/src/cache/test_support.rs | 69 + backend/hls/src/cache/tests/admission.rs | 26 +- backend/hls/src/cache/write_budget.rs | 393 ++++ backend/hls/src/gc.rs | 2 + backend/hls/src/gc/collector.rs | 8 + backend/hls/src/gc/reclaim.rs | 72 +- backend/hls/src/gc/report.rs | 9 +- backend/hls/src/gc/revisions.rs | 141 ++ backend/hls/src/gc/selection.rs | 34 +- backend/hls/src/gc/tests.rs | 1 + backend/hls/src/gc/tests/publication.rs | 101 + backend/hls/src/gc/tests/revisions.rs | 248 +++ backend/hls/src/gc/tests/transport.rs | 4 + backend/hls/src/initial_strip.rs | 18 + backend/hls/src/late_custom_response.rs | 168 ++ backend/hls/src/late_custom_response/tests.rs | 47 + backend/hls/src/lease.rs | 2 + backend/hls/src/lease/identity.rs | 4 +- backend/hls/src/lease/lifecycle.rs | 22 +- backend/hls/src/lease/publication.rs | 11 + backend/hls/src/lease/revision_bindings.rs | 7 + backend/hls/src/lease/runtime_policy.rs | 17 +- backend/hls/src/lease/terminal.rs | 113 +- backend/hls/src/lease/tests/behavior.rs | 4 +- backend/hls/src/lease/timing.rs | 2 + backend/hls/src/lib.rs | 9 + backend/hls/src/manager.rs | 1 + backend/hls/src/manager/activity.rs | 8 +- backend/hls/src/manager/cleanup.rs | 22 - backend/hls/src/manager/config.rs | 5 +- backend/hls/src/manager/publication.rs | 6 +- backend/hls/src/manager/terminal.rs | 96 +- backend/hls/src/manager/tests/support.rs | 14 + backend/hls/src/manager/tests/terminal.rs | 29 + .../hls/src/manifest_acceptance/cohorts.rs | 13 +- .../hls/src/manifest_acceptance/episode.rs | 33 + .../hls/src/manifest_acceptance/evaluation.rs | 139 +- .../hls/src/manifest_acceptance/evidence.rs | 18 + backend/hls/src/manifest_acceptance/tests.rs | 1 + .../manifest_acceptance/tests/evaluation.rs | 163 +- .../tests/reduced_retry.rs | 190 ++ backend/hls/src/manifest_commit.rs | 4 + backend/hls/src/manifest_fetch/episode.rs | 78 +- backend/hls/src/manifest_fetch/fetch.rs | 34 +- backend/hls/src/manifest_fetch/model.rs | 2 + backend/hls/src/manifest_fetch/quality.rs | 3 + backend/hls/src/manifest_fetch/recovery.rs | 242 ++- backend/hls/src/manifest_fetch/tests.rs | 34 + backend/hls/src/map_fetcher.rs | 228 ++- backend/hls/src/media_reserve.rs | 212 +- backend/hls/src/observability.rs | 19 + backend/hls/src/origin.rs | 57 +- backend/hls/src/origin_account_runtime.rs | 205 ++ backend/hls/src/origin_attempt.rs | 360 ++++ backend/hls/src/refresh/commit.rs | 59 +- backend/hls/src/refresh/failure.rs | 2 +- backend/hls/src/refresh/mod.rs | 109 +- backend/hls/src/refresh/short_manifest.rs | 215 ++ backend/hls/src/refresh/switch_staging.rs | 17 +- backend/hls/src/refresh/tests.rs | 3 + backend/hls/src/refresh/tests/acceptance.rs | 2 + .../hls/src/refresh/tests/account_handoff.rs | 139 ++ .../hls/src/refresh/tests/archive_master.rs | 1 + .../src/refresh/tests/headers_redirects.rs | 3 + .../hls/src/refresh/tests/origin_binding.rs | 16 +- .../hls/src/refresh/tests/reduced_retry.rs | 338 ++++ backend/hls/src/refresh/tests/retry_policy.rs | 49 +- .../hls/src/refresh/tests/short_manifest.rs | 669 +++++++ backend/hls/src/refresh/tests/support.rs | 29 +- .../hls/src/refresh/tests/switch_staging.rs | 11 +- backend/hls/src/renderer.rs | 5 +- backend/hls/src/response/tests.rs | 1 + backend/hls/src/runtime_custom_tail.rs | 186 +- backend/hls/src/segment_fetcher.rs | 38 +- backend/hls/src/segment_fetcher/commit.rs | 6 +- backend/hls/src/segment_fetcher/demand.rs | 25 +- .../hls/src/segment_fetcher/key_dependency.rs | 39 +- backend/hls/src/segment_fetcher/origin.rs | 122 +- backend/hls/src/segment_fetcher/scheduler.rs | 63 +- .../hls/src/segment_fetcher/startup_fill.rs | 255 ++- .../hls/src/segment_fetcher/startup_write.rs | 105 + backend/hls/src/segment_fetcher/tests.rs | 1 + .../hls/src/segment_fetcher/tests/behavior.rs | 6 +- .../src/segment_fetcher/tests/lifecycle.rs | 245 ++- .../src/segment_fetcher/tests/persistence.rs | 2 +- .../hls/src/segment_fetcher/tests/retry.rs | 2 +- .../segment_fetcher/tests/startup_capacity.rs | 394 ++++ backend/hls/src/segment_repair.rs | 3 + backend/hls/src/segment_repair/disk_output.rs | 137 ++ backend/hls/src/segment_repair/manager.rs | 74 +- backend/hls/src/segment_repair/probe.rs | 38 +- backend/hls/src/segment_repair/processing.rs | 88 +- .../hls/src/segment_repair/tests/transport.rs | 1 + backend/hls/src/segment_revision.rs | 6 +- backend/hls/src/segment_watchdog.rs | 139 +- backend/hls/src/session.rs | 114 ++ backend/hls/src/short_manifest.rs | 214 ++ backend/hls/src/terminal_commit.rs | 88 +- backend/hls/src/terminal_tail/plan.rs | 1 + backend/hls/src/terminal_tail/tests.rs | 2 + backend/hls/src/timeline.rs | 59 +- backend/hls/src/transient.rs | 5 + backend/hls/src/transient/identity.rs | 11 + backend/hls/src/transient/manifests.rs | 2 + backend/hls/src/transient/resources.rs | 19 + backend/hls/src/transient/tests/streaming.rs | 81 + .../src/transient_fetcher/tests/lifecycle.rs | 8 +- backend/hls/src/transient_manifest.rs | 14 +- backend/iptv/src/stalker/cookie_jar.rs | 2 +- backend/iptv/src/xtream/mod.rs | 18 +- .../pipeline_transparency_tests/transport.rs | 2 +- .../playlist/tests/curation_publication.rs | 8 +- .../tests/curation_publication/item_limit.rs | 4 +- .../processor/playlist/tests/mapping_stage.rs | 6 +- .../tests/quality_rejection_fallback.rs | 22 +- .../m3u_update_quality.rs | 12 +- .../persisted_input_status.rs | 4 +- backend/processing/src/processor/stalker.rs | 2 +- .../src/processor/stalker_refresh.rs | 6 +- backend/repository/src/alias_repository.rs | 4 +- backend/repository/src/lineup_guide_index.rs | 4 +- .../playlist_repository/tests/lifecycle.rs | 2 +- .../playlist_repository/tests/transport.rs | 2 +- .../tests/behavior_quality_acceptance.rs | 6 +- .../src/active_user_manager/connections.rs | 10 +- .../session/src/active_user_manager/mod.rs | 4 +- .../src/active_user_manager/observability.rs | 59 +- .../session/src/active_user_manager/tests.rs | 1 + .../tests/adaptive_streams.rs | 18 +- .../active_user_manager/tests/expiry_gc.rs | 6 +- .../tests/provider_handover.rs | 321 +++ .../tests/request_claims.rs | 4 +- .../tests/socket_activity.rs | 14 +- .../session/src/connection_manager/cleanup.rs | 2 +- .../src/connection_manager/registration.rs | 9 +- .../src/connection_manager/tests/lifecycle.rs | 2 +- backend/session/src/event_manager.rs | 2 +- .../src/event_manager/playlist_update.rs | 4 +- .../shared_stream_manager/tests/lifecycle.rs | 4 +- .../configuration/hls-cache-state-machine.md | 2 +- docs/src/configuration/reverse-proxy.md | 48 +- .../configuration/shared-hls-configuration.md | 20 +- .../configuration/shared-hls-runtime-flow.md | 60 +- .../shared-hls-troubleshooting.md | 38 +- .../app/components/playlist/epg_view/mod.rs | 2 +- .../components/playlist/input_update_card.rs | 6 +- .../playlist/playlist_update_view.rs | 2 +- .../source_editor/output_xtream_form.rs | 4 +- frontend/src/hooks/use_server_status.rs | 73 +- .../src/model/input_update_capabilities.rs | 2 +- frontend/src/model/input_update_card_model.rs | 4 +- .../src/model/pipeline_transparency_model.rs | 14 +- .../src/model/playlist_update_card_status.rs | 6 +- frontend/src/model/staged_completion_tests.rs | 4 +- frontend/src/services/playlist_service.rs | 10 +- shared/src/model/auth/role.rs | 2 +- shared/src/model/config/curation.rs | 2 +- shared/src/model/config/epg/tests.rs | 2 +- shared/src/model/config/target/tests.rs | 2 +- shared/src/model/config/trakt.rs | 2 +- shared/src/model/playlist_update_status.rs | 4 +- shared/src/model/stalker.rs | 2 +- .../src/utils/hash_utils/tests/hashing_ids.rs | 6 +- test/fixtures/hls/reduced-retry/baseline.m3u8 | 16 + test/fixtures/hls/reduced-retry/conflict.m3u8 | 16 + test/fixtures/hls/reduced-retry/repaired.m3u8 | 16 + 247 files changed, 14737 insertions(+), 1778 deletions(-) create mode 100644 backend/app/src/api/endpoints/hls_api/origin_account_runtime.rs create mode 100644 backend/app/src/api/endpoints/hls_api/origin_refresh_request.rs create mode 100644 backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs create mode 100644 backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs create mode 100644 backend/app/src/api/endpoints/hls_api/tests/origin_account_rebind.rs create mode 100644 backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs create mode 100644 backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs create mode 100644 backend/app/tests/hls_origin_account_handover.rs create mode 100644 backend/hls/src/availability/manifest_failure.rs create mode 100644 backend/hls/src/availability/tests/late_custom_response.rs create mode 100644 backend/hls/src/cache/test_support.rs create mode 100644 backend/hls/src/cache/write_budget.rs create mode 100644 backend/hls/src/gc/revisions.rs create mode 100644 backend/hls/src/gc/tests/revisions.rs create mode 100644 backend/hls/src/late_custom_response.rs create mode 100644 backend/hls/src/late_custom_response/tests.rs create mode 100644 backend/hls/src/manifest_acceptance/tests/reduced_retry.rs create mode 100644 backend/hls/src/origin_account_runtime.rs create mode 100644 backend/hls/src/origin_attempt.rs create mode 100644 backend/hls/src/refresh/short_manifest.rs create mode 100644 backend/hls/src/refresh/tests/account_handoff.rs create mode 100644 backend/hls/src/refresh/tests/reduced_retry.rs create mode 100644 backend/hls/src/refresh/tests/short_manifest.rs create mode 100644 backend/hls/src/segment_fetcher/startup_write.rs create mode 100644 backend/hls/src/segment_fetcher/tests/startup_capacity.rs create mode 100644 backend/hls/src/segment_repair/disk_output.rs create mode 100644 backend/hls/src/short_manifest.rs create mode 100644 backend/session/src/active_user_manager/tests/provider_handover.rs create mode 100644 test/fixtures/hls/reduced-retry/baseline.m3u8 create mode 100644 test/fixtures/hls/reduced-retry/conflict.m3u8 create mode 100644 test/fixtures/hls/reduced-retry/repaired.m3u8 diff --git a/.gitignore b/.gitignore index 2e402de12..71e9d3097 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,8 @@ backend/app/Cargo.lock # only exist on a developer's own machine), so it is not project source. /.claude/ /.mcp.json +AGENTS.md +GOAL.md # Local refactoring measurements (clippy inventories, cargo metadata, timings). # These are engineering artifacts of the modularization work, not project docs. diff --git a/Cargo.lock b/Cargo.lock index fafe2b946..f49ed90cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4453,6 +4453,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-stream", + "tokio-tungstenite", "tokio-util", "tower", "tower-http", diff --git a/Cargo.toml b/Cargo.toml index 78ed20cee..572379895 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,14 +24,6 @@ return_self_not_must_use = "allow" missing_errors_doc = "allow" large_futures = "allow" -# Its suggested rewrite of `assert!(x.is_empty())` is -# `assert_eq!(x, [] as [some::crate_root::Path; 0])`, which names a type that -# is usually not in scope at the assertion — which is why `clippy --fix` -# declines to apply it. The panic message improves; every one of the 39 call -# sites gets materially harder to read. Reverse this if the suggestion is -# ever made to use an in-scope type. -assert_is_empty = "allow" - [workspace.dependencies] shared = { version = "3", path = "shared" } tuliprox-hls = { version = "3", path = "backend/hls" } diff --git a/backend/app/Cargo.toml b/backend/app/Cargo.toml index 672eb919a..25570619e 100644 --- a/backend/app/Cargo.toml +++ b/backend/app/Cargo.toml @@ -100,6 +100,7 @@ windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Stora vergen = { version = "10.0.3", features = ["build"] } [dev-dependencies] +tokio-tungstenite = "0.29.0" tuliprox-processing = { workspace = true, features = ["test-support"] } tuliprox-hls = { workspace = true, features = ["test-support"] } # The migration and playlist-stream tests build legacy v2 databases and diff --git a/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs b/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs index 65c189438..f592dbfed 100644 --- a/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs +++ b/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs @@ -195,7 +195,7 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi .await; assert_eq!(retry.admission.permission(), UserConnectionPermission::Allowed); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -326,7 +326,7 @@ async fn resolve_admission_with_strategies_allows_other_channel_after_recent_evi let admission = result.admission; assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -458,7 +458,7 @@ async fn resolve_admission_with_strategies_does_not_suppress_different_session_o assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(grace_mode, None); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -641,7 +641,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same app_state.connection_manager.release_connection(&hls_addr).await; assert_eq!(app_state.active_users.user_connections(&user.username).await, 0); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); let mut close_rx = app_state.connection_manager.get_close_connection_channel(); let result = resolve_admission_with_strategies( @@ -664,7 +664,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(grace_mode, None); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); assert_eq!( tokio::time::timeout(std::time::Duration::from_millis(100), close_rx.recv()).await.ok().and_then(Result::ok), Some(crate::api::model::CloseConnectionSignal::WithReason( diff --git a/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs b/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs index 67f602c30..a9fcceed7 100644 --- a/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs +++ b/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs @@ -128,7 +128,7 @@ async fn create_stream_response_details_preserves_stored_headers_when_fallback_o socket_bound: false, }) .await; - assert!(!created.is_empty()); + assert_ne!(created.len(), 0); app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; let busy = app_state.active_provider.acquire_exact_connection_with_grace( @@ -233,7 +233,7 @@ async fn force_provider_stream_response_clears_stored_headers_after_fallback_ope socket_bound: false, }) .await; - assert!(!created.is_empty()); + assert_ne!(created.len(), 0); app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; let busy = app_state.active_provider.acquire_exact_connection_with_grace( diff --git a/backend/app/src/api/config_file.rs b/backend/app/src/api/config_file.rs index 3a6245b32..9e5762afc 100644 --- a/backend/app/src/api/config_file.rs +++ b/backend/app/src/api/config_file.rs @@ -471,7 +471,7 @@ mod tests { let refreshed = refresh_forced_targets(Arc::clone(¤t), &sources_with(vec![target(1, "my-target")])); assert!(!refreshed.enabled, "a reload must not enable a filter the operator never asked for"); - assert!(refreshed.target_names.is_empty()); + assert_eq!(refreshed.target_names.len(), 0); } /// Target ids are positional, so a surviving name must come back with @@ -502,6 +502,6 @@ mod tests { let refreshed = refresh_forced_targets(forced(&["gone"]), &sources_with(vec![target(1, "other")])); assert!(!refreshed.enabled, "an empty filter must not be left enabled: it would process nothing"); - assert!(refreshed.target_names.is_empty()); + assert_eq!(refreshed.target_names.len(), 0); } } diff --git a/backend/app/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs index 068f0721b..7750be40a 100644 --- a/backend/app/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -21,7 +21,7 @@ use serde::Deserialize; use std::{str::FromStr, sync::Arc}; use tuliprox_hls::api::{ finite_hls_immutable_media_response, resolve_hls_standalone_custom_segment, HlsAccessLeaseId, - HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError, + HlsMediaActivityCommitOutcome, HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError, }; use url::form_urlencoded; @@ -286,7 +286,9 @@ async fn cvs_standalone_hls_segment_response( asset_fingerprint, index, now_ms, - ) { + ) + .await + { Ok(access) => access, Err( HlsStandaloneCustomSegmentError::InvalidIndex @@ -299,13 +301,28 @@ async fn cvs_standalone_hls_segment_response( { return *response; } - finite_hls_immutable_media_response( - access.bytes, + let response = finite_hls_immutable_media_response( + access.bytes.clone(), headers.get(header::RANGE), HLS_CVS_CONTENT_TYPE, HLS_CVS_IMMUTABLE_CACHE_CONTROL, method == Method::HEAD, - ) + ); + if response.status().is_success() { + match app_state + .hls + .proxy + .record_standalone_custom_segment_activity(&access_lease, &access, current_time_millis()) + .await + { + HlsMediaActivityCommitOutcome::Committed => {} + HlsMediaActivityCommitOutcome::StaleLeaseIdentity => return StatusCode::NOT_FOUND.into_response(), + HlsMediaActivityCommitOutcome::DeferredLockContention => { + return super::hls_terminal_response::hls_temporary_resource_unavailable_response(1_000) + } + } + } + response } async fn validate_hls_standalone_custom_access( diff --git a/backend/app/src/api/endpoints/hls_api/manifest/admission.rs b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs index c0b8c946d..f1eb75302 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/admission.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs @@ -1,4 +1,5 @@ use super::{ + super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}, clear_hls_provisioning_handoff_consumer, duration_to_millis_saturating, effective_hls_url_failover_provider_for_fetch_url, hls_access_lease_timing_for_session, hls_access_lease_ttl_ms, hls_availability_reevaluation_registration_failure_response, hls_cache_configured, @@ -29,8 +30,7 @@ use tuliprox_hls::api::{ safe_proxy_session_id, safe_session_key, HlsAccessContext, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, HlsAccessLeaseTouch, HlsAccountBindingProtection, HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, HlsManifestCommitRequirement, HlsOriginIoContext, HlsOriginWorkClass, - HlsPostRefreshRuntime, HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry, - OriginRefreshRequest, ProxySessionId, RetryPolicy, + HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry, ProxySessionId, }; #[allow(clippy::too_many_arguments)] @@ -251,13 +251,10 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest prepared_origin.runtime_provider_config.clone(), )?; { - let mut session_guard = session.write().await; + let session_guard = session.read().await; if session_guard.is_gc_marked_for_removal() { return Some(hls_canonical_retry_after_response()); } - if prepared_origin.origin_account_binding_to_store.is_some() { - session_guard.replace_origin_account_binding(prepared_origin.origin_account_binding_to_store); - } } mark_hls_authorized_manifest_access(app_state, &session, now_ms).await; let selected_account = session.read().await.origin_account_binding.as_ref().map_or_else( @@ -298,7 +295,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest (HlsManifestAcceptanceDirective::none(), Some(owner_key)) } HlsManifestAcceptanceEvaluationOutcome::SessionSuperseded => { - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + drop(prepared_origin.preacquired_origin_account_handle); return Some(hls_canonical_retry_after_response()); } }; @@ -317,7 +314,6 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest }; let origin_policy = hls_effective_origin_acquire_policy(&session).await; - let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone(); let mut preacquired_provider_handle = prepared_origin.preacquired_origin_account_handle; let mut origin_io = HlsOriginIoContext { ctx: hls_ctx.clone(), @@ -328,49 +324,37 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest reservation_ttl_secs, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), }; if availability_reevaluation_owner_key.is_none() { if let Some(provider_handle) = preacquired_provider_handle.take() { - origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + origin_io = origin_io.with_managed_provider_handle(provider_handle); } } - let refresh_request = OriginRefreshRequest { - app_config: Arc::clone(&app_state.app_config), - session: Arc::clone(&session), - origin_entry, - headers, - origin_provider_session_headers, - client: app_state.http_clients.default.load().as_ref().clone(), - no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), - use_manual_redirects: app_state.should_use_manual_redirects(), - segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), - hls_proxy: Arc::clone(&app_state.hls.proxy), - segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), - segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), - map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), - origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), - manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), - strip: app_state.hls.proxy.strip().clone(), - retry_policy: RetryPolicy::default(), - reverse_proxy_rewrite_secret: rewrite_secret.to_vec(), - transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), - manifest_commit_requirement, - fresh_manifest_requirement_generation: None, - acceptance_directive, - access_lease_id: Some(access_lease_id.clone()), - disabled_headers: app_state.get_disabled_headers(), - now_ms, - origin_io: Some(origin_io), - post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: hls_ctx.downgrade() }), - }; + let refresh_request = hls_origin_refresh_request( + app_state, + &session, + tuliprox_hls::api::HlsPreparedOriginAccount { entry: origin_entry, snapshot: prepared_origin.snapshot }, + HlsOriginRefreshOptions { + headers, + origin_io, + manifest_commit_requirement, + acceptance_directive, + access_lease_id: Some(access_lease_id.clone()), + rewrite_secret: rewrite_secret.to_vec(), + now_ms, + }, + ); let refresh_ordering = if session_outcome == HlsSessionStoreOutcome::Reused { refresh_ordering } else { HlsManifestRefreshOrdering::Background }; if let Some(owner_key) = availability_reevaluation_owner_key { - app_state.connection_manager.release_provider_handle(preacquired_provider_handle); + drop(preacquired_provider_handle); touch_initial_manifest_access_lease_window( app_state, access_lease_id, diff --git a/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs index 4864df106..4232efed7 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs @@ -217,7 +217,7 @@ pub(in crate::api::endpoints::hls_api) async fn evaluate_hls_canonical_owner_han return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); } match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) => { + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalCustomResponse(_) => { let Some(response) = hls_terminal_playback_response(&lease, context.proxy_session_id, context.access_lease_id) else { diff --git a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs index 3820bc4d7..d8fc1da7b 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs @@ -482,6 +482,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_plans( pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry( plan: SharedHlsProvisioningSegmentPlan, content_length: u64, + media_start_ms: u64, duration_ms: u64, now_ms: u64, ) -> SegmentEntry { @@ -497,6 +498,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry( host_local_index: u32::try_from(plan.proxy_seq).unwrap_or(u32::MAX), }, proxy_seq: plan.proxy_seq, + media_start_ms, duration_ms, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -602,10 +604,13 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_shared_hls_provisioning_h session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); } session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + let media_start_ms = session_guard.next_media_start_ms; + session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms); session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); - session_guard - .segments - .insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms)); + session_guard.segments.insert( + plan.proxy_seq, + shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms), + ); inserted = true; } if inserted { @@ -648,10 +653,12 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); } session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + let media_start_ms = session_guard.next_media_start_ms; + session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms); session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); session_guard.segments.insert( plan.proxy_seq, - shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms), + shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms), ); } session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); diff --git a/backend/app/src/api/endpoints/hls_api/mod.rs b/backend/app/src/api/endpoints/hls_api/mod.rs index 10c21bf12..928f36d44 100644 --- a/backend/app/src/api/endpoints/hls_api/mod.rs +++ b/backend/app/src/api/endpoints/hls_api/mod.rs @@ -110,10 +110,12 @@ pub(super) struct HlsProxyResourcePathParams { mod catchup; mod manifest; +mod origin_account_runtime; +mod origin_refresh_request; mod owner_token; mod segment; mod session; - +use self::origin_account_runtime::prepare_hls_origin_runtime; pub(in crate::api) use catchup::*; pub(in crate::api) use manifest::*; pub(in crate::api) use owner_token::*; diff --git a/backend/app/src/api/endpoints/hls_api/origin_account_runtime.rs b/backend/app/src/api/endpoints/hls_api/origin_account_runtime.rs new file mode 100644 index 000000000..9691336c7 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/origin_account_runtime.rs @@ -0,0 +1,719 @@ +#![allow(clippy::wildcard_imports)] +use super::{ + origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}, + session::*, + *, +}; + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(super) async fn prepare_hls_origin_runtime( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + work_kind: HlsOriginWorkKind, + work_class: HlsOriginWorkClass, + now_ms: u64, +) -> Result { + { + let current = session.read().await; + if current.origin_source.input_id != input.id + || (!current.origin_source.input_name.is_empty() && current.origin_source.input_name != input.name) + || current.origin_account_binding.as_ref().is_some_and(|binding| binding.input_name != input.name) + { + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: hls_no_account_reason_for_binding(current.origin_account_binding.as_ref()), + }); + } + } + advance_hls_origin_account_lifecycle(app_state, session, now_ms).await; + // Healthy preparation does not queue behind account replacement. Preserve the manifest + // singleflight boundary when many clients arrive while a qualified window awaits media. + if let Some(current_input) = + app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name) + { + let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await); + let binding = observed.binding.clone(); + if let Some(binding) = binding.filter(|binding| { + binding.is_active() + && hls_origin_account_status(&app_state.hls_ctx(), binding) == HlsOriginAccountStatus::Known + }) { + let prepared = prepared_hls_origin_runtime_for_known_binding( + app_state, + ¤t_input, + raw_request_url, + session_entry_url, + &binding, + ) + .await; + let mut current = session.write().await; + if !current.is_gc_marked_for_removal() && observed.matches_session(¤t) { + if current.origin_account_runtime.is_none() { + current.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime { + app_state: Arc::downgrade(app_state), + input_id: input.id, + input_name: Arc::clone(&input.name), + raw_request_url: raw_request_url.to_string(), + entry_url: session_entry_url.to_string(), + fingerprint: fingerprint.clone(), + })); + } + return Ok(PreparedHlsOriginRuntime::published( + prepared, + HlsOriginAccountSnapshot::from_session(¤t), + )); + } + } + } + let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock); + let _prepare_guard = prepare_lock.lock().await; + let Some(input) = app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name) + else { + let mut session = session.write().await; + if session.origin_account_rebind.is_allowed_now(now_ms) { + if let Some(binding) = session.origin_account_binding.clone() { + session.origin_account_rebind.mark_attempt_started(binding.account_name, now_ms); + session.origin_account_rebind.mark_failed(now_ms); + } + } + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: hls_no_account_reason_for_binding(session.origin_account_binding.as_ref()), + }); + }; + session.write().await.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime { + app_state: Arc::downgrade(app_state), + input_id: input.id, + input_name: Arc::clone(&input.name), + raw_request_url: raw_request_url.to_string(), + entry_url: session_entry_url.to_string(), + fingerprint: fingerprint.clone(), + })); + let input = input.as_ref(); + advance_hls_origin_account_lifecycle(app_state, session, now_ms).await; + let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await); + let previous = observed.binding.clone(); + let mut prepared = prepare_hls_origin_runtime_locked( + app_state, + session, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + work_kind, + work_class, + now_ms, + ) + .await?; + let snapshot = publish_hls_origin_account_with_status(app_state, session, |current| { + let snapshot = if let Some(snapshot) = prepared.prepared_snapshot.take() { + if current.is_gc_marked_for_removal() || !snapshot.matches_session(current) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + snapshot + } else { + if current.is_gc_marked_for_removal() || !observed.matches_session(current) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + if let Some(mut binding) = prepared.binding_to_publish.take() { + binding.generation = previous.as_ref().map_or(0, |previous| previous.generation.saturating_add(1)); + binding.provider_binding_tag = prepared + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .and_then(|handle| handle.binding_tag); + if previous.as_ref().is_some_and(|previous| previous.account_name != binding.account_name) { + current.handoff_origin_account(binding); + } else { + current.replace_origin_account_binding(Some(binding)); + } + } + HlsOriginAccountSnapshot::from_session(current) + }; + Ok(snapshot) + }) + .await?; + Ok(PreparedHlsOriginRuntime::published(prepared, snapshot)) +} + +// Lock order: viewer rows, then HLS state. Publication and projection are synchronous once +// both locks are acquired: cancellation cannot publish an account without publishing its UI rows. +pub(super) async fn publish_hls_origin_account_with_status( + app_state: &Arc, + session: &HlsSessionHandle, + publish: impl FnOnce( + &mut tuliprox_hls::api::HlsSession, + ) -> Result, +) -> Result { + let shared_stream_id = hls_cache_shared_stream_id(&session.read().await.proxy_session_id); + let mut projection = app_state.active_users.lock_shared_stream_provider_projection(shared_stream_id).await; + let mut current = session.write().await; + let snapshot = publish(&mut current)?; + if !snapshot.matches_session(¤t) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + if let Some(binding) = snapshot.binding.as_ref().filter(|binding| binding.is_active()) { + projection.update_provider(&binding.account_name); + } + Ok(snapshot) +} + +async fn advance_hls_origin_account_lifecycle(app_state: &Arc, session: &HlsSessionHandle, now_ms: u64) { + promote_elapsed_hls_account_overlaps(app_state, now_ms).await; + detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await; + reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await; +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn prepare_hls_origin_runtime_locked( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + work_kind: HlsOriginWorkKind, + work_class: HlsOriginWorkClass, + now_ms: u64, +) -> Result { + let playback_kind = hls_session_origin_playback_kind(session).await; + let existing_binding = session.read().await.origin_account_binding.clone(); + let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached); + if reacquire_detached_binding && !session.read().await.origin_account_rebind.is_allowed_now(now_ms) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref()); + if reacquire_detached_binding { + log_hls_origin_binding_reacquire_started(session, work_kind).await; + } + if let Some(binding) = existing_binding { + if binding.is_active() + || !matches!(hls_origin_account_status(&app_state.hls_ctx(), &binding), HlsOriginAccountStatus::Known) + { + match hls_origin_account_status(&app_state.hls_ctx(), &binding) { + stale_status @ (HlsOriginAccountStatus::Missing + | HlsOriginAccountStatus::Expired + | HlsOriginAccountStatus::Blocked) => { + return rebind_hls_origin_account( + app_state, + session, + input, + raw_request_url, + session_entry_url, + &binding, + stale_status, + fingerprint, + connection_kind, + priority, + playback_kind, + now_ms, + ) + .await; + } + HlsOriginAccountStatus::Known => { + return Ok(prepared_hls_origin_runtime_for_known_binding( + app_state, + input, + raw_request_url, + session_entry_url, + &binding, + ) + .await); + } + } + } + } + + match prepare_hls_origin_runtime_with_new_account( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + false, + work_kind, + work_class, + playback_kind, + now_ms, + ) + .await + { + Ok(prepared) => { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)), + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} + } + + if work_class.allows_speculative_overlap() { + if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + playback_kind, + now_ms, + ) + .await + { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + + if let Ok(prepared) = prepare_hls_speculative_origin_runtime( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + playback_kind, + now_ms, + ) + .await + { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + } else { + debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value()); + } + + if work_class.allows_grace() { + match prepare_hls_origin_runtime_with_new_account( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + true, + work_kind, + work_class, + playback_kind, + now_ms, + ) + .await + { + Ok(prepared) => { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { + return Err(HlsOriginRuntimeAcquireError::Fatal(status)) + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} + } + } else { + debug!( + "HLS origin account grace skipped: work_class={} reason=background-origin-work", + work_class.as_log_value() + ); + } + + if reacquire_detached_binding { + log_hls_origin_binding_reacquire_failed(session, "no-account-available").await; + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason }) +} + +#[allow(clippy::too_many_arguments)] +#[allow(clippy::too_many_lines)] +pub(super) async fn rebind_hls_origin_account( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + stale_binding: &HlsOriginAccountBinding, + stale_status: HlsOriginAccountStatus, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + playback_kind: PlaybackKind, + now_ms: u64, +) -> Result { + { + let mut session_guard = session.write().await; + if !session_guard.origin_account_rebind.is_allowed_now(now_ms) { + debug!( + "HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms); + } + + let safe_proxy_session = { + let session_guard = session.read().await; + safe_proxy_session_id(&session_guard.proxy_session_id) + }; + debug!( + "HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}", + safe_proxy_session, + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); + app_state.active_provider.clear_identified_provider_reservation( + &stale_binding.session_owner, + &stale_binding.account_name, + stale_binding.provider_binding_tag, + ); + let expected_work_generation = { + let mut session_guard = session.write().await; + if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| { + binding.account_name == stale_binding.account_name + && binding.session_owner == stale_binding.session_owner + && binding.provider_binding_tag == stale_binding.provider_binding_tag + }) { + binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms); + debug!( + "HLS origin binding detached: proxy_session={} account={} reason={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()), + HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason() + ); + } + session_guard.invalidate_queued_origin_work(); + session_guard.activity.origin_work_generation + }; + + let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( + &input.name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)), + ) else { + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await; + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: hls_no_account_reason_for_binding(Some(stale_binding)), + }); + }; + + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_config) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_config()) + else { + drop(managed_handle); + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(fetch_url) = resolve_hls_origin_fetch_url( + &app_state.app_config, + input, + raw_request_url, + session_entry_url, + Some(&provider_config), + ) + .await + else { + drop(managed_handle); + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(new_account_name) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_name()) + else { + drop(managed_handle); + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let mut new_binding = HlsOriginAccountBinding::rebound( + Arc::clone(&input.name), + new_account_name, + stale_binding.session_owner.clone(), + stale_binding.generation.saturating_add(1), + now_ms, + ); + new_binding.provider_binding_tag = managed_handle.handle().and_then(|handle| handle.binding_tag); + let prepared_snapshot = publish_hls_origin_account_with_status(app_state, session, |session_guard| { + if !publish_hls_origin_account_rebind( + session_guard, + stale_binding, + expected_work_generation, + new_binding.clone(), + ) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + Ok(HlsOriginAccountSnapshot::from_session(session_guard)) + }) + .await?; + debug!( + "HLS origin account rebound: old_account={} new_account={}", + sanitize_sensitive_info(stale_binding.account_name.as_ref()), + sanitize_sensitive_info(new_binding.account_name.as_ref()) + ); + + Ok(HlsOriginRuntimeAllocation { + url_failover_provider: hls_url_failover_provider_for_origin_context( + input, + raw_request_url, + session_entry_url, + &fetch_url, + ), + runtime_provider_config: Some(provider_config), + fetch_url, + binding_to_publish: None, + prepared_snapshot: Some(prepared_snapshot), + preacquired_origin_account_handle: Some(managed_handle), + }) +} + +// Resolution may suspend after detaching. Publish only into the same account and work generation. +fn publish_hls_origin_account_rebind( + session: &mut tuliprox_hls::api::HlsSession, + expected: &HlsOriginAccountBinding, + expected_work_generation: u64, + replacement: HlsOriginAccountBinding, +) -> bool { + if session.is_gc_marked_for_removal() + || session.activity.origin_work_generation != expected_work_generation + || !session.origin_account_binding.as_ref().is_some_and(|current| { + current.input_name == expected.input_name + && current.account_name == expected.account_name + && current.session_owner == expected.session_owner + && current.generation == expected.generation + && current.provider_binding_tag == expected.provider_binding_tag + }) + { + return false; + } + session.handoff_origin_account(replacement); + session.origin_account_rebind.mark_success(); + true +} + +pub(super) async fn mark_hls_origin_rebind_failed( + session: &HlsSessionHandle, + stale_binding: &HlsOriginAccountBinding, + now_ms: u64, + reason: &str, +) { + let mut session_guard = session.write().await; + session_guard.origin_account_rebind.mark_failed(now_ms); + debug!( + "HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); +} + +use std::{future::Future, pin::Pin, sync::Weak}; +use tuliprox_hls::api::{ + trigger_origin_refresh_sync, HlsBoundAccountAcquireErrorKind, HlsManifestAcceptanceDirective, + HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount, + LiveHlsOriginEntry, +}; + +struct AppHlsOriginAccountRuntime { + app_state: Weak, + input_id: u16, + input_name: Arc, + raw_request_url: String, + entry_url: String, + fingerprint: Fingerprint, +} + +impl std::fmt::Debug for AppHlsOriginAccountRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AppHlsOriginAccountRuntime").field("input_id", &self.input_id).finish_non_exhaustive() + } +} + +impl HlsOriginAccountRuntime for AppHlsOriginAccountRuntime { + #[allow(clippy::too_many_lines)] + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> Pin> + Send + 'a>> + { + Box::pin(async move { + let app_state = self.app_state.upgrade().ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?; + let input = app_state + .app_config + .get_input_by_id(self.input_id) + .filter(|input| input.name == self.input_name) + .ok_or(HlsBoundAccountAcquireErrorKind::Missing)?; + let (proxy_id, policy, unavailable) = { + let session = session.read().await; + ( + session.proxy_session_id.clone(), + session.effective_origin_acquire_policy_or_default(), + session.is_gc_marked_for_removal(), + ) + }; + if unavailable + || !app_state + .hls + .proxy + .segment_worker_pool() + .access_leases() + .write() + .await + .has_usable_access_lease_for_session(&proxy_id, current_time_millis()) + { + return Err(HlsBoundAccountAcquireErrorKind::Unavailable); + } + let prepared = prepare_hls_origin_runtime( + &app_state, + session, + &input, + &self.raw_request_url, + &self.entry_url, + &proxy_id, + &self.fingerprint, + policy.connection_kind, + policy.priority, + if work == HlsOriginAccountWork::Manifest { + HlsOriginWorkKind::Manifest + } else { + HlsOriginWorkKind::Resource + }, + class, + current_time_millis(), + ) + .await + .map_err(|_| HlsBoundAccountAcquireErrorKind::Unavailable)?; + // Drop releases unused allocations synchronously, including cancellation after prepare. + let managed = prepared.preacquired_origin_account_handle; + let entry = LiveHlsOriginEntry::parse_with_provider_configs( + &prepared.fetch_url, + prepared.url_failover_provider, + prepared.runtime_provider_config, + ) + .ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?; + let snapshot = prepared.snapshot; + let (pending, origin_headers) = { + let current = session.read().await; + if current.is_gc_marked_for_removal() || !snapshot.matches_session(¤t) { + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + (current.origin_account_manifest_pending.is_some(), current.origin_request_headers.clone()) + }; + drop(managed); + if work == HlsOriginAccountWork::Media && pending { + let origin_io = HlsOriginIoContext { + ctx: app_state.hls_ctx(), + client_addr: self.fingerprint.addr, + allow_grace: class.allows_grace(), + priority: policy.priority, + connection_kind: policy.connection_kind, + reservation_ttl_secs: session.read().await.account_overlap_timing().reservation_ttl_secs(), + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), + }; + let manifest_commit_requirement = tuliprox_hls::api::hls_manifest_commit_requirement( + session, + tuliprox_hls::api::HlsSessionStoreOutcome::Reused, + None, + false, + current_time_millis(), + ) + .await; + let request = hls_origin_refresh_request( + &app_state, + session, + HlsPreparedOriginAccount { entry: entry.clone(), snapshot: snapshot.clone() }, + HlsOriginRefreshOptions { + headers: origin_headers, + origin_io, + manifest_commit_requirement, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + rewrite_secret: app_state.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + trigger_origin_refresh_sync(request).await; + if session.read().await.origin_account_manifest_pending.is_some() { + return Err(HlsBoundAccountAcquireErrorKind::Unavailable); + } + } + Ok(HlsPreparedOriginAccount { entry, snapshot }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tuliprox_hls::api::{HlsSession, HlsSessionKey}; + + #[test] + fn rebind_publication_rejects_superseded_work_binding_and_gc() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "publication"), b"secret", 0); + let expected = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("expired"), &session.proxy_session_id, 0); + session.replace_origin_account_binding(Some(expected.clone())); + let replacement = HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("replacement"), + expected.session_owner.clone(), + 1, + 1, + ); + session.invalidate_queued_origin_work(); + assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 0, replacement.clone())); + assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name, expected.account_name); + assert!(session.origin_account_manifest_pending.is_none()); + session.origin_account_binding.as_mut().unwrap().generation = 7; + assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone())); + assert_eq!(session.origin_account_binding.as_ref().unwrap().generation, 7); + session.replace_origin_account_binding(Some(expected.clone())); + session.mark_for_gc_removal(); + assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone())); + session.clear_gc_removal_mark(); + assert!(publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement)); + assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "replacement"); + assert_eq!(session.activity.origin_work_generation, 2); + assert_eq!(session.origin_account_manifest_pending, Some(1)); + } +} diff --git a/backend/app/src/api/endpoints/hls_api/origin_refresh_request.rs b/backend/app/src/api/endpoints/hls_api/origin_refresh_request.rs new file mode 100644 index 000000000..211f4b7b2 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/origin_refresh_request.rs @@ -0,0 +1,58 @@ +use super::{AppState, HlsOriginIoContext, HlsSessionHandle}; +use axum::http::HeaderMap; +use std::sync::Arc; +use tuliprox_hls::api::{ + HlsAccessLeaseId, HlsManifestAcceptanceDirective, HlsManifestCommitRequirement, HlsPostRefreshRuntime, + HlsPreparedOriginAccount, OriginRefreshRequest, RetryPolicy, +}; + +pub(super) struct HlsOriginRefreshOptions { + pub headers: HeaderMap, + pub origin_io: HlsOriginIoContext, + pub manifest_commit_requirement: HlsManifestCommitRequirement, + pub acceptance_directive: HlsManifestAcceptanceDirective, + pub access_lease_id: Option, + pub rewrite_secret: Vec, + pub now_ms: u64, +} + +pub(super) fn hls_origin_refresh_request( + app_state: &Arc, + session: &HlsSessionHandle, + prepared: HlsPreparedOriginAccount, + options: HlsOriginRefreshOptions, +) -> OriginRefreshRequest { + let mut origin_io = options.origin_io; + let origin_provider_session_headers = prepared.snapshot.provider_session_headers.clone(); + origin_io.prepared_account = Some(prepared.snapshot); + OriginRefreshRequest { + app_config: Arc::clone(&app_state.app_config), + session: Arc::clone(session), + origin_entry: prepared.entry, + headers: options.headers, + origin_provider_session_headers, + disabled_headers: app_state.get_disabled_headers(), + client: app_state.http_clients.default.load().as_ref().clone(), + no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), + use_manual_redirects: app_state.should_use_manual_redirects(), + segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), + hls_proxy: Arc::clone(&app_state.hls.proxy), + segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), + segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), + map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), + origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), + manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), + strip: app_state.hls.proxy.strip().clone(), + retry_policy: RetryPolicy::default(), + reverse_proxy_rewrite_secret: options.rewrite_secret, + transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), + manifest_commit_requirement: options.manifest_commit_requirement, + fresh_manifest_requirement_generation: None, + refresh_identity: None, + acceptance_directive: options.acceptance_directive, + access_lease_id: options.access_lease_id, + now_ms: options.now_ms, + origin_io: Some(origin_io), + post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: app_state.hls_ctx().downgrade() }), + } +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/access.rs b/backend/app/src/api/endpoints/hls_api/segment/access.rs index 89f18c52b..f3766b16f 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/access.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/access.rs @@ -186,7 +186,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_lease_allows_cached_segment( match &lease.playback_mode { HlsLeasePlaybackMode::Live => true, HlsLeasePlaybackMode::TerminalTail(plan) => plan.protected_base_proxy_seqs.contains(&proxy_seq), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => false, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => false, } } diff --git a/backend/app/src/api/endpoints/hls_api/segment/fallback.rs b/backend/app/src/api/endpoints/hls_api/segment/fallback.rs index fca2112b1..dbc7af969 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/fallback.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/fallback.rs @@ -94,7 +94,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_runtime_or_standalone_custom return fallback_status.into_response(); }; match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { + HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) { return response; } @@ -196,7 +198,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_origin_runtime_resource_failure_re err: HlsOriginRuntimeAcquireError, ) -> axum::response::Response { match err { - HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => StatusCode::SERVICE_UNAVAILABLE.into_response(), + HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => { + hls_canonical_status_response(StatusCode::SERVICE_UNAVAILABLE) + } HlsOriginRuntimeAcquireError::Fatal(status) => hls_canonical_status_response(status), } } diff --git a/backend/app/src/api/endpoints/hls_api/segment/registration.rs b/backend/app/src/api/endpoints/hls_api/segment/registration.rs index 935159283..c4106f79c 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/registration.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/registration.rs @@ -27,6 +27,10 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_hls_cache_stream_register access: &HlsAccessContext, session: &HlsSessionHandle, ) -> Option { + // Share the existing origin-publication gate: a registration cannot retain an old + // account snapshot across a handover and then overwrite its projected provider. + let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock); + let _prepare_guard = prepare_lock.lock().await; let (proxy_session_id, origin_source, origin_account_binding) = { let session = session.read().await; if session.is_gc_marked_for_removal() { diff --git a/backend/app/src/api/endpoints/hls_api/segment/resources.rs b/backend/app/src/api/endpoints/hls_api/segment/resources.rs index f9c68d7ea..4b12d2c85 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/resources.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/resources.rs @@ -145,9 +145,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_resource( match &lease.playback_mode { HlsLeasePlaybackMode::Live => serve_hls_live_transient_resource(context).await, HlsLeasePlaybackMode::TerminalTail(_) => serve_hls_terminal_key_resource(context, &lease.playback_mode).await, - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { - StatusCode::NOT_FOUND.into_response() - } + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => StatusCode::NOT_FOUND.into_response(), } } @@ -226,7 +226,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc }; let resource = cache_resolution.resource; let origin_headers = cache_resolution.origin_headers; - let origin_provider_session_headers = cache_resolution.origin_provider_session_headers; let cache_action = cache_resolution.action; match cache_action { @@ -266,7 +265,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc resource, cache_action, origin_headers, - origin_provider_session_headers, cache_duration_ms, }) .await @@ -277,21 +275,14 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientPassthroughContext<'a> pub(in crate::api::endpoints::hls_api) resource: TransientResourceRef, pub(in crate::api::endpoints::hls_api) cache_action: HlsTransientObjectCacheAction, pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, - pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64, } pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_resource( context: HlsTransientPassthroughContext<'_>, ) -> axum::response::Response { - let HlsTransientPassthroughContext { - endpoint, - resource, - cache_action, - origin_headers, - origin_provider_session_headers, - cache_duration_ms, - } = context; + let HlsTransientPassthroughContext { endpoint, resource, cache_action, origin_headers, cache_duration_ms } = + context; if let HlsTransientObjectCacheAction::FetchAndCache(fetch_token) = cache_action { return fetch_and_cache_transient_origin_response(HlsTransientEndpointCacheFetchContext { app_state: endpoint.app_state, @@ -304,7 +295,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r resource_file: endpoint.resource_file, fetch_token: *fetch_token, origin_headers, - origin_provider_session_headers, range_header: endpoint.range_header, cache_duration_ms, }) @@ -321,7 +311,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r resource: &resource, resource_file: &endpoint.resource_file, origin_headers, - origin_provider_session_headers, range_header: endpoint.range_header.clone(), policy: policy.clone(), }) diff --git a/backend/app/src/api/endpoints/hls_api/segment/segments.rs b/backend/app/src/api/endpoints/hls_api/segment/segments.rs index 59393842f..f80c72b60 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/segments.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/segments.rs @@ -229,6 +229,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_terminal_segment( } HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => None, }, @@ -318,6 +319,8 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context( fingerprint: &Fingerprint, preacquired_provider_handle: Option, ) -> SegmentFetchContext { + let preacquired_provider_handle = preacquired_provider_handle + .map(|handle| tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle)); let (headers, origin_provider_session_headers, origin_policy, reservation_ttl_secs) = { let session = session.read().await; ( @@ -336,9 +339,12 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context( reservation_ttl_secs, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), }; if let Some(provider_handle) = preacquired_provider_handle { - origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + origin_io = origin_io.with_managed_provider_handle(provider_handle); } SegmentFetchContext { session: Arc::clone(session), diff --git a/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs b/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs index 82ffb062e..e8730f553 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs @@ -35,7 +35,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointOriginFetchReq pub(in crate::api::endpoints::hls_api) resource: &'a TransientResourceRef, pub(in crate::api::endpoints::hls_api) resource_file: &'a TransientResourceFile, pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, - pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, pub(in crate::api::endpoints::hls_api) range_header: Option, pub(in crate::api::endpoints::hls_api) policy: SegmentFetchPolicy, } @@ -54,14 +53,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_ no_redirect_client: request.app_state.http_clients.no_redirect.load().as_ref().clone(), use_manual_redirects: request.app_state.should_use_manual_redirects(), }; - let log_identity = { + let (log_identity, generation, provider_headers) = { let session = request.session.read().await; - HlsLogIdentity::from_session(&session) + ( + HlsLogIdentity::from_session(&session), + session.activity.origin_work_generation, + session.origin_provider_session_headers.clone(), + ) }; let fetch_request = HlsTransientOriginFetchRequest { resolved_origin_uri: request.resource.resolved_origin_uri.clone(), origin_headers: request.origin_headers, - origin_provider_session_headers: request.origin_provider_session_headers, + origin_provider_session_headers: provider_headers, range_header: request.range_header, resource_file: request.resource_file.clone(), resource_kind: request.resource.kind, @@ -77,12 +80,19 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_ request.fingerprint, request.headers, &runtime_prepare_error, + HlsTransientOriginBindingSnapshot { resource: request.resource.clone(), generation }, ); let result = fetch_hls_transient_origin_response_with_attempt_prepare(fetch_request, prepare_attempt).await; let runtime_prepare_error = *runtime_prepare_error.lock().await; HlsTransientOriginFetchResult { result, runtime_prepare_error } } +#[derive(Clone)] +pub(in crate::api::endpoints::hls_api) struct HlsTransientOriginBindingSnapshot { + resource: TransientResourceRef, + generation: u64, +} + /// Builds the shared per-attempt prepare closure for transient origin fetches. /// Runtime acquire failures are captured in `runtime_prepare_error` and mapped /// to a provider-unavailable fetch error so the retry loop can proceed uniformly. @@ -93,6 +103,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure( fingerprint: &Fingerprint, headers: &HeaderMap, runtime_prepare_error: &Arc>>, + snapshot: HlsTransientOriginBindingSnapshot, ) -> impl FnMut( HlsResourceFetchAttempt, ) -> futures::future::BoxFuture< @@ -112,6 +123,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure( let fingerprint = fingerprint.clone(); let headers = headers.clone(); let runtime_prepare_error = Arc::clone(&runtime_prepare_error); + let snapshot = snapshot.clone(); async move { match prepare_hls_transient_origin_io_for_authorized_resource_work( &app_state, @@ -123,7 +135,17 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure( ) .await { - Ok(guard) => Ok(guard), + Ok(guard) => { + let session = session.read().await; + if session.activity.origin_work_generation != snapshot.generation + || !session.transient.origin_resource_is_fetchable(&snapshot.resource, current_time_millis()) + { + return Err(HlsOriginResourceFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::StoreRace, + )); + } + Ok(guard) + } Err(err) => { *runtime_prepare_error.lock().await = Some(err); Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Unavailable)) @@ -146,7 +168,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointCacheFetchCont pub(in crate::api::endpoints::hls_api) resource_file: TransientResourceFile, pub(in crate::api::endpoints::hls_api) fetch_token: TransientObjectFetchToken, pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, - pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, pub(in crate::api::endpoints::hls_api) range_header: Option, pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64, } @@ -298,14 +319,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin no_redirect_client: context.app_state.http_clients.no_redirect.load().as_ref().clone(), use_manual_redirects: context.app_state.should_use_manual_redirects(), }; - let log_identity = { + let (log_identity, generation, provider_headers) = { let session = context.session.read().await; - HlsLogIdentity::from_session(&session) + ( + HlsLogIdentity::from_session(&session), + session.activity.origin_work_generation, + session.origin_provider_session_headers.clone(), + ) }; let fetch_request = HlsTransientOriginFetchRequest { resolved_origin_uri: context.resource.resolved_origin_uri.clone(), origin_headers: context.origin_headers.clone(), - origin_provider_session_headers: context.origin_provider_session_headers.clone(), + origin_provider_session_headers: provider_headers, range_header: None, resource_file: context.resource_file.clone(), resource_kind: context.resource.kind, @@ -336,6 +361,7 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin context.fingerprint, context.headers, &runtime_prepare_error, + HlsTransientOriginBindingSnapshot { resource: context.resource.clone(), generation }, ); let final_failure = match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(cache_fetch_request, prepare_attempt) diff --git a/backend/app/src/api/endpoints/hls_api/session.rs b/backend/app/src/api/endpoints/hls_api/session.rs index c19a53c7a..3347980da 100644 --- a/backend/app/src/api/endpoints/hls_api/session.rs +++ b/backend/app/src/api/endpoints/hls_api/session.rs @@ -378,13 +378,37 @@ pub(super) fn hls_url_failover_provider_for_origin_context( } pub(super) struct PreparedHlsOriginRuntime { + pub(super) fetch_url: String, + pub(super) runtime_provider_config: Option>, + pub(super) url_failover_provider: Option>, + pub(super) snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot, + pub(super) preacquired_origin_account_handle: Option, +} + +impl PreparedHlsOriginRuntime { + pub(super) fn published( + allocation: HlsOriginRuntimeAllocation, + snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot, + ) -> Self { + Self { + fetch_url: allocation.fetch_url, + runtime_provider_config: allocation.runtime_provider_config, + url_failover_provider: allocation.url_failover_provider, + snapshot, + preacquired_origin_account_handle: allocation.preacquired_origin_account_handle, + } + } +} + +pub(super) struct HlsOriginRuntimeAllocation { pub(super) fetch_url: String, pub(super) runtime_provider_config: Option>, // URL failover comes from source.yml provider:// resolution. Origin-account // binding/handles are runtime account reservations and must stay separate. pub(super) url_failover_provider: Option>, - pub(super) origin_account_binding_to_store: Option, - pub(super) preacquired_origin_account_handle: Option, + pub(super) binding_to_publish: Option, + pub(super) prepared_snapshot: Option, + pub(super) preacquired_origin_account_handle: Option, } pub(super) fn effective_hls_url_failover_provider_for_fetch_url( @@ -626,194 +650,8 @@ pub(super) fn hls_soft_overlap_delay_ms( /// /// Archive-backed sessions must keep catchup semantics instead of plain live HLS; /// live sessions keep the live HLS kind. -async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind { - if session.read().await.key.archive_reference.is_some() { - PlaybackKind::Catchup - } else { - PlaybackKind::LiveHls - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(super) async fn prepare_hls_origin_runtime( - app_state: &Arc, - session: &HlsSessionHandle, - input: &ConfigInput, - raw_request_url: &str, - session_entry_url: &str, - proxy_session_id: &ProxySessionId, - fingerprint: &Fingerprint, - connection_kind: crate::api::model::ConnectionKind, - priority: i8, - work_kind: HlsOriginWorkKind, - work_class: HlsOriginWorkClass, - now_ms: u64, -) -> Result { - let playback_kind = hls_session_origin_playback_kind(session).await; - promote_elapsed_hls_account_overlaps(app_state, now_ms).await; - detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await; - reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await; - let existing_binding = session.read().await.origin_account_binding.clone(); - let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached); - let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref()); - if reacquire_detached_binding { - log_hls_origin_binding_reacquire_started(session, work_kind).await; - } - if let Some(binding) = existing_binding { - if binding.is_active() { - match hls_origin_account_status(&app_state.hls_ctx(), &binding) { - stale_status @ (HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired) => { - return rebind_hls_origin_account( - app_state, - session, - input, - raw_request_url, - session_entry_url, - &binding, - stale_status, - fingerprint, - connection_kind, - priority, - playback_kind, - now_ms, - ) - .await; - } - HlsOriginAccountStatus::Known => { - return Ok(prepared_hls_origin_runtime_for_known_binding( - app_state, - input, - raw_request_url, - session_entry_url, - &binding, - ) - .await); - } - } - } - } - - match prepare_hls_origin_runtime_with_new_account( - app_state, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - false, - work_kind, - work_class, - playback_kind, - now_ms, - ) - .await - { - Ok(prepared) => { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)), - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} - } - - if work_class.allows_speculative_overlap() { - if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime( - app_state, - session, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - playback_kind, - now_ms, - ) - .await - { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - - if let Ok(prepared) = prepare_hls_speculative_origin_runtime( - app_state, - session, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - playback_kind, - now_ms, - ) - .await - { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - } else { - debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value()); - } - - if work_class.allows_grace() { - match prepare_hls_origin_runtime_with_new_account( - app_state, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - true, - work_kind, - work_class, - playback_kind, - now_ms, - ) - .await - { - Ok(prepared) => { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { - return Err(HlsOriginRuntimeAcquireError::Fatal(status)) - } - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} - } - } else { - debug!( - "HLS origin account grace skipped: work_class={} reason=background-origin-work", - work_class.as_log_value() - ); - } - - if reacquire_detached_binding { - log_hls_origin_binding_reacquire_failed(session, "no-account-available").await; - } - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason }) +pub(super) async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind { + session.read().await.origin_playback_kind() } #[allow(clippy::too_many_arguments)] @@ -831,7 +669,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( work_class: HlsOriginWorkClass, playback_kind: PlaybackKind, now_ms: u64, -) -> Result { +) -> Result { let session_owner = build_hls_origin_session_owner(proxy_session_id); let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( &input.name, @@ -852,8 +690,13 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( }); }; + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_handle) = managed_handle.handle() else { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -866,7 +709,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -876,7 +719,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( &provider_handle.allocation, now_ms, ) else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -896,7 +739,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( grace_state ); - Ok(PreparedHlsOriginRuntime { + Ok(HlsOriginRuntimeAllocation { url_failover_provider: hls_url_failover_provider_for_origin_context( input, raw_request_url, @@ -905,8 +748,9 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( ), runtime_provider_config: Some(provider_config), fetch_url, - origin_account_binding_to_store: Some(binding), - preacquired_origin_account_handle: Some(provider_handle), + binding_to_publish: Some(binding), + prepared_snapshot: None, + preacquired_origin_account_handle: Some(managed_handle), }) } @@ -916,7 +760,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding( raw_request_url: &str, session_entry_url: &str, binding: &HlsOriginAccountBinding, -) -> PreparedHlsOriginRuntime { +) -> HlsOriginRuntimeAllocation { let runtime_provider_config = app_state.active_provider.find_provider_config(&binding.account_name); let fetch_url = resolve_hls_origin_fetch_url( &app_state.app_config, @@ -928,7 +772,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding( .await .unwrap_or_else(|| session_entry_url.to_string()); - PreparedHlsOriginRuntime { + HlsOriginRuntimeAllocation { runtime_provider_config, url_failover_provider: hls_url_failover_provider_for_origin_context( input, @@ -937,7 +781,8 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding( &fetch_url, ), fetch_url, - origin_account_binding_to_store: None, + binding_to_publish: None, + prepared_snapshot: None, preacquired_origin_account_handle: None, } } @@ -995,14 +840,15 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A timing.target_duration_ms ); let starting_window_ms = tuliprox_session::STARTING_LEASE_TTL_SECS.saturating_mul(1000); - let media_window_expired = session_guard.activity.last_delivered_media_at_ms.map_or_else( - || now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms), - |last| { - now_ms - > last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms) - }, - ); - if !media_window_expired + // A retained session can contain delivery from an earlier playback. + // Give each newly published binding its own bounded start window; + // healthy preparation leaves pinned_at_ms unchanged. + let starting_window_expired = now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms); + let media_window_expired = session_guard.activity.last_delivered_media_at_ms.is_none_or(|last| { + now_ms > last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms) + }); + if !starting_window_expired + || !media_window_expired || session_guard.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { continue; @@ -1033,7 +879,6 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(super) async fn prepare_hls_origin_policy_preempt_runtime( app_state: &Arc, - new_session: &HlsSessionHandle, input: &ConfigInput, raw_request_url: &str, session_entry_url: &str, @@ -1043,7 +888,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( priority: i8, playback_kind: PlaybackKind, now_ms: u64, -) -> Result { +) -> Result { let request_policy = HlsEffectiveOriginAcquirePolicy::new(connection_kind, priority, now_ms); let Some(candidate) = find_hls_origin_policy_preempt_candidate(app_state, input, proxy_session_id, request_policy, now_ms).await @@ -1087,8 +932,13 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( debug!("HLS origin policy preemption denied: reason=exact-acquire-failed"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_handle) = managed_handle.handle() else { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=missing-provider-config"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1102,7 +952,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=invalid-origin-url"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1113,7 +963,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( &provider_handle.allocation, now_ms, ) else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=invalid-allocation"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1140,16 +990,12 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( } } if !detached_victim { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=stale-candidate"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); } - { - let mut session_guard = new_session.write().await; - session_guard.replace_origin_account_binding(Some(binding.clone())); - } debug!( "HLS origin policy preempted: account={} victim_proxy_session={} winner_proxy_session={} victim_kind={:?} victim_priority={} request_kind={:?} request_priority={}", sanitize_sensitive_info(candidate.account_name.as_ref()), @@ -1167,7 +1013,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( HlsOriginAccountDetachedReason::PreemptedByHigherPriority.as_log_reason() ); - Ok(PreparedHlsOriginRuntime { + Ok(HlsOriginRuntimeAllocation { url_failover_provider: hls_url_failover_provider_for_origin_context( input, raw_request_url, @@ -1176,8 +1022,9 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( ), runtime_provider_config: Some(provider_config), fetch_url, - origin_account_binding_to_store: Some(binding), - preacquired_origin_account_handle: Some(provider_handle), + binding_to_publish: Some(binding), + prepared_snapshot: None, + preacquired_origin_account_handle: Some(managed_handle), }) } @@ -1283,7 +1130,6 @@ pub(super) fn hls_origin_policy_preempt_candidate_is_better( #[allow(clippy::too_many_arguments)] pub(super) async fn prepare_hls_speculative_origin_runtime( app_state: &Arc, - new_session: &HlsSessionHandle, input: &ConfigInput, raw_request_url: &str, session_entry_url: &str, @@ -1293,7 +1139,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( priority: i8, playback_kind: PlaybackKind, now_ms: u64, -) -> Result { +) -> Result { let Some(candidate) = find_hls_account_overlap_candidate(app_state, input, proxy_session_id, now_ms).await else { debug!("HLS account overlap denied: reason=no-soft-active-candidate"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1316,8 +1162,13 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( debug!("HLS account overlap denied: reason=speculative-acquire-failed"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_handle) = managed_handle.handle() else { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS account overlap denied: reason=missing-provider-config"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1331,7 +1182,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS account overlap denied: reason=invalid-origin-url"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1344,10 +1195,6 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( candidate.reclaim_until_ms, now_ms, ); - { - let mut session_guard = new_session.write().await; - session_guard.replace_origin_account_binding(Some(binding.clone())); - } debug!( "HLS account overlap granted: account={} victim_proxy_session={} winner_proxy_session={} reclaim_until_ms={} eligible_after_ms={} delay_ms={} tuliprox_target_user_connections={} origin_input_account_connections={}", sanitize_sensitive_info(candidate.account_name.as_ref()), @@ -1359,7 +1206,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( candidate.tuliprox_target_user_connection_capacity, candidate.origin_input_account_connection_capacity ); - Ok(PreparedHlsOriginRuntime { + Ok(HlsOriginRuntimeAllocation { url_failover_provider: hls_url_failover_provider_for_origin_context( input, raw_request_url, @@ -1368,8 +1215,9 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( ), runtime_provider_config: Some(provider_config), fetch_url, - origin_account_binding_to_store: Some(binding), - preacquired_origin_account_handle: Some(provider_handle), + binding_to_publish: Some(binding), + prepared_snapshot: None, + preacquired_origin_account_handle: Some(managed_handle), }) } @@ -1650,154 +1498,6 @@ pub(super) async fn promote_elapsed_hls_account_overlaps(app_state: &Arc, - session: &HlsSessionHandle, - input: &ConfigInput, - raw_request_url: &str, - session_entry_url: &str, - stale_binding: &HlsOriginAccountBinding, - stale_status: HlsOriginAccountStatus, - fingerprint: &Fingerprint, - connection_kind: crate::api::model::ConnectionKind, - priority: i8, - playback_kind: PlaybackKind, - now_ms: u64, -) -> Result { - { - let mut session_guard = session.write().await; - if !session_guard.origin_account_rebind.is_allowed_now(now_ms) { - debug!( - "HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000", - safe_proxy_session_id(&session_guard.proxy_session_id), - sanitize_sensitive_info(stale_binding.account_name.as_ref()) - ); - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - } - session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms); - } - - let safe_proxy_session = { - let session_guard = session.read().await; - safe_proxy_session_id(&session_guard.proxy_session_id) - }; - debug!( - "HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}", - safe_proxy_session, - sanitize_sensitive_info(stale_binding.account_name.as_ref()) - ); - app_state.active_provider.clear_identified_provider_reservation( - &stale_binding.session_owner, - &stale_binding.account_name, - stale_binding.provider_binding_tag, - ); - { - let mut session_guard = session.write().await; - if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| { - binding.account_name == stale_binding.account_name - && binding.session_owner == stale_binding.session_owner - && binding.provider_binding_tag == stale_binding.provider_binding_tag - }) { - binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms); - debug!( - "HLS origin binding detached: proxy_session={} account={} reason={}", - safe_proxy_session_id(&session_guard.proxy_session_id), - sanitize_sensitive_info(stale_binding.account_name.as_ref()), - HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason() - ); - } - session_guard.invalidate_queued_origin_work(); - } - - let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( - &input.name, - &fingerprint.addr, - false, - priority, - connection_kind, - Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)), - ) else { - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await; - return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { - reason: hls_no_account_reason_for_binding(Some(stale_binding)), - }); - }; - - let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await; - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - - let Some(fetch_url) = resolve_hls_origin_fetch_url( - &app_state.app_config, - input, - raw_request_url, - session_entry_url, - Some(&provider_config), - ) - .await - else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await; - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - - let Some(new_account_name) = provider_handle.allocation.get_provider_name() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await; - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - - let new_binding = HlsOriginAccountBinding::rebound( - Arc::clone(&input.name), - new_account_name, - stale_binding.session_owner.clone(), - stale_binding.generation.saturating_add(1), - now_ms, - ); - { - let mut session_guard = session.write().await; - session_guard.replace_origin_account_binding(Some(new_binding.clone())); - session_guard.origin_account_rebind.mark_success(); - } - debug!( - "HLS origin account rebound: old_account={} new_account={}", - sanitize_sensitive_info(stale_binding.account_name.as_ref()), - sanitize_sensitive_info(new_binding.account_name.as_ref()) - ); - - Ok(PreparedHlsOriginRuntime { - url_failover_provider: hls_url_failover_provider_for_origin_context( - input, - raw_request_url, - session_entry_url, - &fetch_url, - ), - runtime_provider_config: Some(provider_config), - fetch_url, - origin_account_binding_to_store: None, - preacquired_origin_account_handle: Some(provider_handle), - }) -} - -pub(super) async fn mark_hls_origin_rebind_failed( - session: &HlsSessionHandle, - stale_binding: &HlsOriginAccountBinding, - now_ms: u64, - reason: &str, -) { - let mut session_guard = session.write().await; - session_guard.origin_account_rebind.mark_failed(now_ms); - debug!( - "HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000", - safe_proxy_session_id(&session_guard.proxy_session_id), - sanitize_sensitive_info(stale_binding.account_name.as_ref()) - ); -} - #[allow(clippy::too_many_arguments)] pub(super) async fn prepare_hls_cache_user_session( app_state: &Arc, @@ -1995,6 +1695,7 @@ pub(super) async fn hls_segment_request_requires_origin_work( && entry.origin_fetch_ref.is_some() } +#[cfg(test)] pub(super) async fn hls_origin_binding_needs_reacquire(session: &HlsSessionHandle) -> bool { let session = session.read().await; session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached) @@ -2008,7 +1709,7 @@ pub(super) fn hls_transient_origin_binding_requires_runtime_prepare( || (binding.is_active() && matches!( hls_origin_account_status(hls_ctx, binding), - HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired + HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired | HlsOriginAccountStatus::Blocked )) } @@ -2021,11 +1722,22 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work( work_kind: HlsOriginWorkKind, now_ms: u64, ) -> Result, HlsOriginRuntimeAcquireError> { - if !hls_origin_binding_needs_reacquire(session).await { + let runtime = session.read().await.origin_account_runtime.clone(); + if let Some(runtime) = runtime { + runtime + .prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand) + .await + .map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?; return Ok(None); } - if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + if !session + .read() + .await + .origin_account_binding + .as_ref() + .is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&app_state.hls_ctx(), binding)) + { + return Ok(None); } let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) .await @@ -2047,13 +1759,17 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work( now_ms, ) .await?; - if let Some(binding) = prepared_origin.origin_account_binding_to_store { - session.write().await.replace_origin_account_binding(Some(binding)); + drop(prepared_origin); + let runtime = session.read().await.origin_account_runtime.clone(); + if let Some(runtime) = runtime { + runtime + .prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand) + .await + .map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?; } - Ok(prepared_origin.preacquired_origin_account_handle) + Ok(None) } -#[allow(clippy::too_many_lines)] pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work( app_state: &Arc, session: &HlsSessionHandle, @@ -2062,120 +1778,55 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work req_headers: &HeaderMap, now_ms: u64, ) -> Result, HlsOriginRuntimeAcquireError> { - let hls_ctx = app_state.hls_ctx(); - let existing_binding = session.read().await.origin_account_binding.clone(); - let origin_policy = hls_effective_origin_acquire_policy(session).await; - let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(session).await; - if let Some(binding) = existing_binding.as_ref().filter(|binding| binding.is_active()) { - match hls_origin_account_status(&hls_ctx, binding) { - HlsOriginAccountStatus::Known => { - let origin_io = HlsOriginIoContext { - ctx: hls_ctx.clone(), - client_addr: fingerprint.addr, - allow_grace: HlsOriginWorkClass::Demand.allows_grace(), - priority: origin_policy.priority, - connection_kind: origin_policy.connection_kind, - reservation_ttl_secs, - preacquired_provider_handle: None, - started_generation: None, - }; - if let Ok(lease_guard) = begin_hls_origin_account_io_bounded( - &origin_io, - session, - binding, - hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms), - ) - .await - { - let (started_generation, active_origin_work_count) = { - let mut session = session.write().await; - let count = Arc::clone(&session.activity.active_origin_work_count); - (session.start_origin_work(), count) - }; - return Ok(Some(HlsTransientOriginIoGuard::new( - Arc::clone(session), - active_origin_work_count, - origin_io, - lease_guard, - started_generation, - ))); - } - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - } - HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired => {} - } - } - - if !existing_binding - .as_ref() - .is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, binding)) - { - return Ok(None); - } - if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - } - let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) - .await - .map_err(HlsOriginRuntimeAcquireError::Fatal)?; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let prepared_origin = prepare_hls_origin_runtime( + prepare_hls_origin_binding_for_authorized_resource_work( app_state, session, - &request_context.input, - &request_context.hls_url, - request_context.session_entry_url.as_str(), - &proxy_session_id, + access_context, fingerprint, - origin_policy.connection_kind, - origin_policy.priority, + req_headers, HlsOriginWorkKind::Resource, - HlsOriginWorkClass::Demand, now_ms, ) .await?; - if let Some(binding) = prepared_origin.origin_account_binding_to_store { - session.write().await.replace_origin_account_binding(Some(binding)); - } - let Some(provider_handle) = prepared_origin.preacquired_origin_account_handle else { + let Some(binding) = session.read().await.origin_account_binding.clone() else { return Ok(None); }; - let Some(binding) = session.read().await.origin_account_binding.clone().filter(HlsOriginAccountBinding::is_active) - else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + if binding.is_detached() { return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; + } + let policy = hls_effective_origin_acquire_policy(session).await; let origin_io = HlsOriginIoContext { - ctx: hls_ctx, + ctx: app_state.hls_ctx(), client_addr: fingerprint.addr, allow_grace: HlsOriginWorkClass::Demand.allows_grace(), - priority: origin_policy.priority, - connection_kind: origin_policy.connection_kind, - reservation_ttl_secs, + priority: policy.priority, + connection_kind: policy.connection_kind, + reservation_ttl_secs: hls_origin_account_reservation_ttl_secs_for_session(session).await, preacquired_provider_handle: None, started_generation: None, - } - .with_preacquired_provider_handle(provider_handle); - let Ok(lease_guard) = begin_hls_origin_account_io_bounded( + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), + }; + let lease_guard = begin_hls_origin_account_io_bounded( &origin_io, session, &binding, hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms), ) .await - else { - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - let (started_generation, active_origin_work_count) = { + .map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?; + let (generation, active_count) = { let mut session = session.write().await; + if !session.origin_account_binding.as_ref().is_some_and(|current| { + current.generation == binding.generation + && current.account_name == binding.account_name + && current.is_active() + }) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } let count = Arc::clone(&session.activity.active_origin_work_count); (session.start_origin_work(), count) }; - Ok(Some(HlsTransientOriginIoGuard::new( - Arc::clone(session), - active_origin_work_count, - origin_io, - lease_guard, - started_generation, - ))) + Ok(Some(HlsTransientOriginIoGuard::new(Arc::clone(session), active_count, origin_io, lease_guard, generation))) } diff --git a/backend/app/src/api/endpoints/hls_api/tests.rs b/backend/app/src/api/endpoints/hls_api/tests.rs index 7d0a36fec..092208599 100644 --- a/backend/app/src/api/endpoints/hls_api/tests.rs +++ b/backend/app/src/api/endpoints/hls_api/tests.rs @@ -78,15 +78,20 @@ use tuliprox_hls::api::{ HlsCachedManifestOptions, HlsLeaseManifestSnapshotInput, LiveHlsOriginEntry, }; +mod origin_account_rebind; mod owner_token; +mod reduced_retry; +mod same_account_resume; mod archive; mod bandwidth; mod encrypted_media; mod entry_identity; mod headers; +mod late_custom_response; mod lease_lifecycle; mod manifest_entry_freshness; +mod manifest_short_window; mod manifest_startup; mod observability; mod owner_handoff; diff --git a/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs index 08b417e9c..212ff5264 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs @@ -163,7 +163,7 @@ async fn hls_cache_entry_returns_master_playlist_without_origin_refresh_or_sessi .is_some()); assert!(app_state.hls.proxy.sessions().get_by_key(&session_key).await.is_none()); assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -398,7 +398,7 @@ async fn legacy_hls_token_route_renders_channel_unavailable_inline_when_target_h assert!(!response.headers().contains_key(header::LOCATION)); assert_eq!(response.headers()[header::CONTENT_TYPE], "application/vnd.apple.mpegurl"); assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -424,7 +424,7 @@ async fn legacy_hls_token_route_with_invalid_token_returns_bad_request_when_targ assert_eq!(response.status(), StatusCode::BAD_REQUEST); assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -545,7 +545,7 @@ async fn hls_entry_origin_reservation_blocks_foreign_sessions_only_after_media_c assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); assert!(reservation.selected_provider_config.is_some()); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); app_state.connection_manager.release_provider_handle(reservation.provider_handle); // A manifest-only entry reservation is still unconfirmed, so it must not block a @@ -1053,7 +1053,7 @@ async fn hls_entry_for_input_in_maintenance_opens_no_session() { // No maintenance clip is configured here, so the configured fallback status answers. assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert!(app_state.hls.proxy.access_leases().read().await.is_empty(), "no lease for a maintenance input"); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs b/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs new file mode 100644 index 000000000..552348c2e --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs @@ -0,0 +1,259 @@ +use super::*; + +#[tokio::test] +async fn late_custom_response_serves_canonical_manifest_and_immutable_media_without_origin_requests() { + let fixture = terminal_tail::publication_late_fixture().await; + let now_ms = super::super::current_time_millis(); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("published live lease"); + let snapshot = lease.last_manifest_snapshot.expect("published snapshot"); + { + let mut session = fixture.session.write().await; + session.segments.clear(); + session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; + } + let requests_before = fixture.origin.manifest_request_count(); + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest"); + assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", snapshot.last_proxy_seq + 1))); + assert!(body.contains(&format!("#EXT-X-TARGETDURATION:{}\n", snapshot.target_duration_ms / 1_000))); + assert!(body.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n")); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + assert!(!body.contains("/hls/shared/live/")); + let media_uris = body.lines().filter(|line| line.contains("/cvs/hls/")).collect::>(); + assert_eq!(media_uris.len(), 12); + let (first, second) = tokio::join!( + get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None), + get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None), + ); + for response in [first, second] { + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await, body.as_bytes()); + } + let media = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await; + assert_eq!(media.status(), StatusCode::OK); + assert_eq!(media.headers()[header::CONTENT_TYPE], "video/mp2t"); + let bytes = response_body(media).await; + assert!(!bytes.is_empty()); + let range = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], Some("bytes=0-187")).await; + assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response_body(range).await, bytes.slice(..188)); + let router = + crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(Arc::clone(&fixture.app_state)); + let head = router + .oneshot( + Request::builder() + .method(Method::HEAD) + .uri(media_uris[0]) + .extension(ConnectInfo(test_addr())) + .body(Body::empty()) + .expect("HEAD request"), + ) + .await + .expect("HEAD response"); + assert_eq!(head.status(), StatusCode::OK); + assert_eq!(head.headers()[header::CONTENT_LENGTH], bytes.len().to_string()); + assert!(response_body(head).await.is_empty()); + fixture.app_state.app_config.custom_stream_response.store(None); + let replay = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await; + assert_eq!(replay.status(), StatusCode::OK); + assert_eq!(response_body(replay).await, bytes); + let wrong_lease = media_uris[0].replace(&fixture.access_lease_id.0, "unknown-lease"); + assert_eq!( + custom_media_response(Arc::clone(&fixture.app_state), &wrong_lease, None).await.status(), + StatusCode::NOT_FOUND + ); + let wrong_index = media_uris[0].replace("/0.ts", "/12.ts"); + assert_eq!( + custom_media_response(Arc::clone(&fixture.app_state), &wrong_index, None).await.status(), + StatusCode::NOT_FOUND + ); + let late = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("terminal lease"); + assert!(matches!(late.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + assert_eq!(late.playback_cursor, lease.playback_cursor); + assert!(!fixture.session.read().await.has_terminal_tail_protections()); + assert_eq!(fixture.origin.manifest_request_count(), requests_before); +} + +#[tokio::test] +async fn late_custom_response_without_configured_asset_retains_error_response() { + let fixture = terminal_tail::publication_late_fixture().await; + fixture.app_state.app_config.custom_stream_response.store(None); + { + let mut session = fixture.session.write().await; + session.segments.clear(); + session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; + } + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response_body(response).await.is_empty()); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("terminal unavailable lease"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); +} + +#[tokio::test] +async fn late_custom_media_get_head_and_range_renew_lease_without_manifest_polls() { + let (fixture, media_uri) = committed_late_http_fixture().await; + for (method, range, status) in [ + (Method::GET, None, StatusCode::OK), + (Method::HEAD, None, StatusCode::OK), + (Method::GET, Some("bytes=0-187"), StatusCode::PARTIAL_CONTENT), + ] { + let now_ms = super::super::current_time_millis(); + let previous_access_ms = now_ms.saturating_sub(50_000); + assert!(fixture + .app_state + .hls + .proxy + .activate_access_lease( + &fixture.access_lease_id, + &fixture.proxy_session_id, + previous_access_ms, + HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + let before = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("valid late lease"); + let response = + custom_media_response_with_method(Arc::clone(&fixture.app_state), &media_uri, range, method).await; + assert_eq!(response.status(), status); + let after = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("renewed late lease"); + assert!(after.last_seen_at_ms >= now_ms); + assert!(after.valid_until_ms > before.valid_until_ms); + assert!(after.active_until_ms > before.active_until_ms); + assert_eq!(after.playback_mode, before.playback_mode); + assert_eq!(after.playback_cursor, before.playback_cursor); + assert_eq!(after.admission_generation, before.admission_generation); + } +} + +#[tokio::test] +async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease() { + let (fixture, media_uri) = committed_late_http_fixture().await; + let now_ms = super::super::current_time_millis(); + assert!(fixture + .app_state + .hls + .proxy + .activate_access_lease( + &fixture.access_lease_id, + &fixture.proxy_session_id, + now_ms - 50_000, + HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + let before = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("valid late lease"); + let fingerprint = media_uri.rsplit('/').nth(1).expect("asset fingerprint"); + for (uri, range, expected) in [ + (media_uri.replace("/0.ts", "/12.ts"), None, StatusCode::NOT_FOUND), + (media_uri.replace(fingerprint, "wrong-fingerprint"), None, StatusCode::NOT_FOUND), + (media_uri.replace(&fixture.access_lease_id.0, "unknown-lease"), None, StatusCode::NOT_FOUND), + (media_uri.clone(), Some("bytes=1000000000000-"), StatusCode::RANGE_NOT_SATISFIABLE), + ] { + let response = custom_media_response(Arc::clone(&fixture.app_state), &uri, range).await; + assert_eq!(response.status(), expected); + } + fixture.app_state.app_config.api_proxy.store(None); + let response = custom_media_response(Arc::clone(&fixture.app_state), &media_uri, None).await; + assert_eq!(response.status(), fixture.app_state.app_config.get_auth_error_status()); + let after = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("unchanged late lease"); + assert_eq!(after.last_seen_at_ms, before.last_seen_at_ms); + assert_eq!(after.valid_until_ms, before.valid_until_ms); + assert_eq!(after.active_until_ms, before.active_until_ms); +} + +async fn committed_late_http_fixture() -> (terminal_tail::PublicationLateFixture, String) { + let fixture = terminal_tail::publication_late_fixture().await; + { + let mut session = fixture.session.write().await; + session.segments.clear(); + session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; + } + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest"); + let media_uri = body.lines().find(|line| line.contains("/cvs/hls/")).expect("custom segment URI").to_owned(); + (fixture, media_uri) +} + +async fn custom_media_response(app_state: Arc, uri: &str, range: Option<&str>) -> Response { + custom_media_response_with_method(app_state, uri, range, Method::GET).await +} + +async fn custom_media_response_with_method( + app_state: Arc, + uri: &str, + range: Option<&str>, + method: Method, +) -> Response { + let router = crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(app_state); + let mut request = Request::builder().method(method).uri(uri); + if let Some(range) = range { + request = request.header(header::RANGE, range); + } + router + .oneshot(request.extension(ConnectInfo(test_addr())).body(Body::empty()).expect("custom media request")) + .await + .expect("custom media response") +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs index ae6bf363c..db7809f38 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs @@ -17,11 +17,14 @@ use std::sync::{ const ORIGIN_WINDOW_SEGMENTS: u64 = 6; const SEGMENT_BODY: &[u8] = &[0x47; 8_192]; -fn origin_playlist(head: u64) -> Vec { +fn origin_playlist(head: u64) -> Vec { origin_playlist_with_duration(head, 2) } + +pub(super) fn origin_playlist_with_duration(head: u64, target_duration: u64) -> Vec { use std::fmt::Write; - let mut body = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + let mut body = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{target_duration}\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); for sequence in head..head + ORIGIN_WINDOW_SEGMENTS { - let _ = write!(body, "#EXTINF:2.0,\nseg-{sequence}.ts\n"); + let _ = write!(body, "#EXTINF:{target_duration}.0,\nseg-{sequence}.ts\n"); } body.into_bytes() } @@ -61,6 +64,20 @@ async fn first_origin_sequence_for_new_lease( request_url: &str, lease: &str, ) -> u64 { + let body = open_manifest_for_new_lease(app_state, input, request_url, lease).await; + let session_key = super::super::build_hls_origin_source(input, "12345").session_key(); + let first_proxy_seq = manifest_media_sequence(&body); + let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session"); + let session = session.read().await; + session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence +} + +pub(super) async fn open_manifest_for_new_lease( + app_state: &Arc, + input: &ConfigInput, + request_url: &str, + lease: &str, +) -> String { let origin_source = super::super::build_hls_origin_source(input, "12345"); let session_key = origin_source.session_key(); let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); @@ -88,11 +105,7 @@ async fn first_origin_sequence_for_new_lease( .await .expect("hls cache should handle the live entrypoint"); assert_eq!(response.status(), StatusCode::OK, "lease {lease}"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); - let first_proxy_seq = manifest_media_sequence(&body); - let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session"); - let session = session.read().await; - session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence + String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8") } /// Moves the retained session's last origin confirmation `idle_ms` into the diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs new file mode 100644 index 000000000..b2192e10f --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs @@ -0,0 +1,526 @@ +use super::{support::TestEncodedManifestOrigin, *}; +use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode}; +use std::sync::atomic::AtomicU64; +use tuliprox_hls::api::HlsFreshManifestRequiredReason; + +struct ShortWindowFixture { + _temp_dir: tempfile::TempDir, + origin: TestSegmentOrigin, + app_state: Arc, + input: ConfigInput, + request_url: String, + head: Arc, + count: Arc, + proxy_session_id: ProxySessionId, +} + +enum ShortWindowResources { + Plain, + EncryptedWithMap, + FirstReady { media_base_url: String }, +} + +async fn short_window_fixture(count: usize) -> ShortWindowFixture { + short_window_resource_fixture(count, ShortWindowResources::Plain).await +} + +async fn short_window_resource_fixture(count: usize, resources: ShortWindowResources) -> ShortWindowFixture { + let startup_mode = match &resources { + ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => HlsStartupMode::Conservative, + ShortWindowResources::FirstReady { .. } => HlsStartupMode::FirstReady, + }; + let head = Arc::new(AtomicU64::new(100)); + let count = Arc::new(AtomicUsize::new(count)); + let (origin_head, origin_count) = (Arc::clone(&head), Arc::clone(&count)); + let origin = spawn_test_binary_origin(Arc::new(move |path| { + let body = if path_has_extension(path, "m3u8") { + let head = origin_head.load(Ordering::SeqCst); + let mut body = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + let extension = match &resources { + ShortWindowResources::Plain | ShortWindowResources::FirstReady { .. } => "ts", + ShortWindowResources::EncryptedWithMap => { + body.push_str("#EXT-X-MAP:URI=\"init.mp4\"\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",IV=0x01010101010101010101010101010101\n"); + "m4s" + } + }; + for offset in 0..origin_count.load(Ordering::SeqCst) { + let segment_url = match &resources { + ShortWindowResources::FirstReady { media_base_url } => { + format!("{media_base_url}/segment-{}.ts", head + u64::try_from(offset).unwrap()) + } + ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => { + format!("segment-{}.{extension}", head + u64::try_from(offset).unwrap()) + } + }; + writeln!(body, "#EXTINF:2.5,\n{segment_url}").unwrap(); + } + Arc::from(body.into_bytes()) + } else if path_has_extension(path, "mp4") { + Arc::from(&b"short-window-map"[..]) + } else if path_has_extension(path, "bin") { + Arc::from(AES_TEST_KEY_BYTES) + } else { + match &resources { + ShortWindowResources::Plain => Arc::from(&b"short-window-media"[..]), + ShortWindowResources::FirstReady { .. } => unreachable!("media uses the gated segment origin"), + ShortWindowResources::EncryptedWithMap => Arc::from(encrypt_test_aes128_cbc_pkcs7( + b"short-window-media", AES_TEST_KEY_BYTES, [1; 16])), + } + }; + TestBinaryOriginResponse::new(StatusCode::OK, body) + })) + .await; + let temp_dir = tempfile::tempdir().unwrap(); + let input = ConfigInput { + id: 1, + name: Arc::from("short-window-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)), + vec![Arc::new(input.clone())], + ); + let mut cache = HlsCacheConfigDto { + cache_path: Some(temp_dir.path().to_string_lossy().to_string()), + startup: HlsStartupConfigDto { mode: startup_mode, ..HlsStartupConfigDto::default() }, + ..HlsCacheConfigDto::default() + }; + cache.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Friendly; + let config = Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(cache), + ..Default::default() + })), + ..Config::default() + }; + app_state.app_config.config.store(Arc::new(config)); + app_state.hls.proxy.update_config(&app_state.app_config).await; + configure_default_test_server(&app_state); + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_string(); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/channel/index.m3u8", origin.base_url); + cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "short-channel", &request_url)).await; + let key = super::super::build_hls_origin_source(&input, "12345").session_key(); + let proxy_session_id = crate::api::model::build_proxy_session_id(&key, &app_state.get_encrypt_secret()); + ShortWindowFixture { _temp_dir: temp_dir, origin, app_state, input, request_url, head, count, proxy_session_id } +} + +/// Keeps the first media fetch pending until all lease entries have been checked. +async fn gated_segment_origin(gate: Arc) -> TestEncodedManifestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let origin_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let requests = Arc::clone(&origin_requests); + let gate = Arc::clone(&gate); + tokio::spawn(async move { + let mut buffer = [0_u8; 2048]; + let read = socket.read(&mut buffer).await.unwrap(); + requests.lock().await.push(String::from_utf8_lossy(&buffer[..read]).to_string()); + let _permit = gate.acquire().await; + let body = [0x47; 8192]; + let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + socket.write_all(headers.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + }); + } + }); + TestEncodedManifestOrigin { base_url: format!("http://{addr}"), requests, task } +} + +async fn wait_for_shared_first_ready_refresh( + fixture: &ShortWindowFixture, + session: &HlsSessionHandle, + previous_requests: usize, +) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if fixture.origin.manifest_request_count() > previous_requests + && !session.read().await.origin_refresh.in_flight + { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("additional leases complete one shared freshness request"); + assert_eq!(fixture.origin.manifest_request_count(), previous_requests + 1); +} + +async fn assert_first_ready_lease_media( + fixture: &ShortWindowFixture, + leases: &mut tokio::task::JoinSet, + minimum_media_sequence: u64, +) { + tokio::time::timeout(Duration::from_secs(5), async { + while let Some(result) = leases.join_next().await { + let body = result.unwrap(); + assert!(manifest_media_sequence(&body) >= minimum_media_sequence); + assert_eq!(media_uri_count(&body), 1); + for uri in body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x47; 8192]); + } + } + }) + .await + .expect("media publication releases every waiting lease"); +} + +#[tokio::test] +async fn first_ready_leases_share_qualification_while_first_segment_is_loading() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let media = gated_segment_origin(Arc::clone(&gate)).await; + let fixture = Arc::new( + short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() }) + .await, + ); + let mut leases = tokio::task::JoinSet::new(); + let first = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&first, "first-ready-0").await }); + let session = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if !media.requests.lock().await.is_empty() { + if let Some(session) = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await + { + if !session.read().await.origin_refresh.in_flight { + break session; + } + } + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("qualified origin has started its first segment fetch"); + let generation = { + let session = session.read().await; + assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady); + assert!(session.last_rendered_manifest.is_none()); + assert_eq!(session.origin_control.acceptance_generation.0, 1); + session.origin_control.acceptance_generation + }; + assert_eq!(fixture.origin.manifest_request_count(), 3); + for lease in 1..20 { + let fixture = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&fixture, &format!("first-ready-{lease}")).await }); + } + wait_for_shared_first_ready_refresh(&fixture, &session, 3).await; + assert!(session.read().await.last_rendered_manifest.is_none()); + assert_eq!(session.read().await.origin_control.acceptance_generation, generation); + gate.close(); + assert_first_ready_lease_media(&fixture, &mut leases, 0).await; + assert_eq!(session.read().await.origin_control.acceptance_generation, generation); +} + +#[tokio::test] +async fn first_ready_revalidation_keeps_its_qualification_while_new_media_is_loading() { + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + let media = gated_segment_origin(Arc::clone(&gate)).await; + let fixture = Arc::new( + short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() }) + .await, + ); + let previous_body = open_short_lease(&fixture, "revalidation-baseline").await; + let session = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap(); + let previous_commit = { + let mut session = session.write().await; + assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady); + assert_eq!(session.origin_control.acceptance_generation.0, 1); + session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0; + session.activity.last_authorized_media_at_ms = None; + session.startup.as_mut().unwrap().revisions.clear(); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Expired; + } + session.last_normal_manifest_commit_identity().unwrap() + }; + assert_eq!(fixture.origin.manifest_request_count(), 3); + assert_eq!(gate.forget_permits(1), 1); + assert_eq!( + hls_manifest_commit_requirement( + &session, + tuliprox_hls::api::HlsSessionStoreOutcome::Reused, + None, + true, + super::super::current_time_millis(), + ) + .await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + } + ); + fixture.head.store(101, Ordering::SeqCst); + let mut leases = tokio::task::JoinSet::new(); + let first = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&first, "revalidation-waiting-0").await }); + let revalidation = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if media.requests.lock().await.len() >= 2 { + let state = session.read().await; + if !state.origin_refresh.in_flight && state.origin_seq_highwater == Some(101) { + break; + } + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await; + let media_requests = media.requests.lock().await.len(); + { + let session = session.read().await; + assert!( + revalidation.is_ok(), + "revalidation requests={} generation={} highwater={:?} media_requests={media_requests} in_flight={} commit={:?}", + fixture.origin.manifest_request_count(), + session.origin_control.acceptance_generation.0, + session.origin_seq_highwater, + session.origin_refresh.in_flight, + session.last_normal_manifest_commit_identity(), + ); + } + assert_eq!(fixture.origin.manifest_request_count(), 5); + { + let session = session.read().await; + assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady); + assert_eq!(session.origin_control.acceptance_generation.0, 2); + assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit)); + } + assert_eq!( + hls_manifest_commit_requirement( + &session, + tuliprox_hls::api::HlsSessionStoreOutcome::Reused, + None, + true, + super::super::current_time_millis(), + ) + .await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + } + ); + for lease in 1..20 { + let fixture = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&fixture, &format!("revalidation-waiting-{lease}")).await }); + } + wait_for_shared_first_ready_refresh(&fixture, &session, 5).await; + assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2); + assert_eq!(session.read().await.last_normal_manifest_commit_identity(), Some(previous_commit)); + gate.close(); + assert_first_ready_lease_media(&fixture, &mut leases, manifest_media_sequence(&previous_body) + 1).await; + assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2); + assert!(session.read().await.last_normal_manifest_commit_identity().unwrap() > previous_commit); +} + +async fn open_short_lease(fixture: &ShortWindowFixture, lease: &str) -> String { + let access_lease_id = HlsAccessLeaseId(lease.to_string()); + let access = test_hls_access_context(fixture.proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&fixture.app_state, &fixture.proxy_session_id, &access_lease_id).await; + let response = super::super::try_hls_cache_canonical_manifest_response( + &fixture.app_state, + &test_fingerprint(), + &access, + &fixture.proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &fixture.request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&fixture.request_url), + input: &fixture.input, + origin_source: super::super::build_hls_origin_source(&fixture.input, "12345"), + }, + HeaderMap::new(), + None, + "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("shared HLS response"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + assert!(!body.contains("#EXT-X-ENDLIST"), "short origin must remain live: {body}"); + assert!(body.contains("/hls/shared/live/")); + body +} + +async fn assert_short_media(fixture: &ShortWindowFixture, body: &str) { + for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), b"short-window-media"); + } +} + +#[tokio::test] +async fn one_segment_origin_publishes_live_manifest_and_segment() { + let fixture = short_window_fixture(1).await; + let body = open_short_lease(&fixture, "short-one").await; + assert_eq!(media_uri_count(&body), 1); + assert_short_media(&fixture, &body).await; + assert_eq!(fixture.origin.manifest_request_count(), 3); +} + +#[tokio::test] +async fn two_segment_origin_publishes_live_manifest_and_segments() { + let fixture = short_window_fixture(2).await; + let body = open_short_lease(&fixture, "short-two").await; + assert_eq!(media_uri_count(&body), 2); + assert_short_media(&fixture, &body).await; + assert_eq!(fixture.origin.manifest_request_count(), 3); +} + +#[tokio::test] +async fn twenty_actual_leases_share_the_same_short_origin_qualification() { + let mut counts = Vec::new(); + for leases in [1, 20] { + let fixture = Arc::new(short_window_fixture(1).await); + let mut tasks = tokio::task::JoinSet::new(); + for lease in 0..leases { + let fixture = Arc::clone(&fixture); + tasks.spawn(async move { open_short_lease(&fixture, &format!("short-shared-{lease}")).await }); + } + while let Some(result) = tasks.join_next().await { + let body = result.unwrap(); + assert_eq!(media_uri_count(&body), 1); + assert_short_media(&fixture, &body).await; + } + counts.push(fixture.origin.manifest_request_count()); + let before_new_lease = fixture.origin.manifest_request_count(); + open_short_lease(&fixture, "later-short-lease").await; + assert_eq!(fixture.origin.manifest_request_count(), before_new_lease); + } + assert_eq!(counts, vec![3, 3]); +} + +#[tokio::test] +async fn ten_short_refresh_cycles_keep_serving_live_media_and_immutable_old_routes() { + assert_ten_short_refresh_cycles([3, 1, 3, 1, 3, 1, 3, 1, 3, 1]).await; +} + +#[tokio::test] +async fn continuously_advancing_one_segment_origin_serves_ten_refresh_cycles() { + assert_ten_short_refresh_cycles([1; 10]).await; +} + +async fn assert_ten_short_refresh_cycles(counts: [usize; 10]) { + let fixture = short_window_fixture(1).await; + let initial = open_short_lease(&fixture, "short-reloads").await; + assert_short_media(&fixture, &initial).await; + let uri = format!("/hls/shared/live/{}/short-reloads/manifest.m3u8", fixture.proxy_session_id.0); + let session = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap(); + let mut previous = initial.clone(); + let mut previous_last = session.read().await.last_rendered_manifest.as_ref().unwrap().last_proxy_seq; + for cycle in 1..=10 { + fixture.head.store(100 + cycle * 4, Ordering::SeqCst); + fixture.count.store(counts[usize::try_from(cycle - 1).unwrap()], Ordering::SeqCst); + { + let mut session = session.write().await; + session.origin_refresh.next_fetch_allowed_at_ms = 0; + } + let mut current = None; + for _ in 0..100 { + let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + assert!(!body.contains("#EXT-X-ENDLIST")); + if body != previous { + current = Some(body); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let body = current.expect("normal refresh advances the short window"); + assert!(manifest_media_sequence(&body) >= manifest_media_sequence(&previous)); + previous = body.clone(); + assert_short_media(&fixture, &body).await; + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &HlsAccessLeaseId("short-reloads".to_string()), + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .unwrap(); + let snapshot = lease.last_manifest_snapshot.unwrap(); + assert!(snapshot.last_proxy_seq > previous_last); + previous_last = snapshot.last_proxy_seq; + assert_eq!(snapshot.playlist_duration_ms, u64::try_from(media_uri_count(&body)).unwrap() * 2_500); + } + assert_eq!(fixture.origin.manifest_request_count(), 13); + assert_short_media(&fixture, &initial).await; + let invalid = uri.replace("short-reloads", "missing-lease"); + assert_ne!(get_response(Arc::clone(&fixture.app_state), &invalid, None).await.status(), StatusCode::OK); +} + +#[tokio::test] +async fn short_manifests_keep_map_key_and_segment_routes_on_later_publications() { + let fixture = short_window_resource_fixture(2, ShortWindowResources::EncryptedWithMap).await; + let initial = open_short_lease(&fixture, "short-resources").await; + let session = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap(); + for uri in initial.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + response_body(response).await; + } + let mut bodies = vec![initial]; + fixture.head.store(102, Ordering::SeqCst); + session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + let uri = format!("/hls/shared/live/{}/short-resources/manifest.m3u8", fixture.proxy_session_id.0); + for _ in 0..100 { + let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + if body != bodies[0] { + bodies.push(body); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(bodies.len(), 2, "a second short publication must advance"); + for body in bodies { + assert!(!body.contains("#EXT-X-ENDLIST")); + for (tag, expected) in [("#EXT-X-MAP:", &b"short-window-map"[..]), ("#EXT-X-KEY:", AES_TEST_KEY_BYTES)] { + let uri = body + .lines() + .find(|line| line.starts_with(tag)) + .unwrap() + .split_once("URI=\"") + .unwrap() + .1 + .split_once('"') + .unwrap() + .0; + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), expected); + } + for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_body(response).await.as_ref(), + encrypt_test_aes128_cbc_pkcs7(b"short-window-media", AES_TEST_KEY_BYTES, [1; 16]) + ); + } + } + assert_eq!(fixture.origin.manifest_request_count(), 4); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs index fd28c77e8..4b5153080 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs @@ -420,7 +420,7 @@ async fn hls_cache_manifest_cold_start_synchronously_returns_initial_manifest() assert_eq!(binding.input_name.as_ref(), "test-input"); assert_eq!(binding.account_name.as_ref(), "test-input"); } - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/app/src/api/endpoints/hls_api/tests/origin_account_rebind.rs b/backend/app/src/api/endpoints/hls_api/tests/origin_account_rebind.rs new file mode 100644 index 000000000..7e03aba87 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/origin_account_rebind.rs @@ -0,0 +1,1725 @@ +use super::{ + super::{prepare_hls_origin_runtime, HlsOriginWorkClass, HlsOriginWorkKind}, + *, +}; +use tuliprox_core::{model::ProviderAccountObservation, utils::current_time_millis}; +use tuliprox_hls::api::{ + hls_origin_account_status, HlsOriginAccountStatus, HlsOriginAccountWork, HlsSegmentFile, OriginMapKey, +}; + +#[derive(Clone, Copy, Debug)] +enum InvalidAccount { + ExpiredRoot, + ExpiredAlias, + RemovedAlias, + DisabledRoot, + DisabledAlias, + RuntimeRoot, + RuntimeAlias, +} + +fn reload_inputs(app: &Arc, inputs: Vec>) { + app.app_config.sources.store(Arc::new(SourcesConfig { + group_lookup: build_group_lookup(&inputs), + inputs, + ..app.app_config.sources.load_full().as_ref().clone() + })); + app.active_provider.update_config(&app.app_config); +} + +fn invalidate_account( + app: &Arc, + input: &mut ConfigInput, + invalid: InvalidAccount, +) -> (&'static str, HlsOriginAccountStatus) { + match invalid { + InvalidAccount::ExpiredRoot => input.exp_date = Some(1), + InvalidAccount::ExpiredAlias => input.aliases.as_mut().unwrap()[0].exp_date = Some(1), + InvalidAccount::RemovedAlias => input.aliases = None, + InvalidAccount::DisabledRoot => input.account_disabled = true, + InvalidAccount::DisabledAlias => input.aliases.as_mut().unwrap()[0].enabled = false, + InvalidAccount::RuntimeRoot | InvalidAccount::RuntimeAlias => { + let name = if matches!(invalid, InvalidAccount::RuntimeRoot) { + &input.name + } else { + &input.aliases.as_ref().unwrap()[0].name + }; + let config = app.active_provider.find_provider_config(name).unwrap(); + assert!(app + .active_provider + .observe_account(ProviderAccountObservation { + name: Arc::clone(name), + identity: config.account_identity(), + status: Some(shared::model::ProxyUserStatus::Banned), + exp_date: None, + }) + .is_some()); + } + } + if !matches!(invalid, InvalidAccount::RuntimeRoot | InvalidAccount::RuntimeAlias) { + reload_inputs(app, vec![Arc::new(input.clone())]); + } + match invalid { + InvalidAccount::ExpiredRoot => ("overlap-input", HlsOriginAccountStatus::Expired), + InvalidAccount::ExpiredAlias => ("account-a", HlsOriginAccountStatus::Expired), + InvalidAccount::RemovedAlias | InvalidAccount::DisabledAlias => ("account-a", HlsOriginAccountStatus::Missing), + InvalidAccount::DisabledRoot | InvalidAccount::RuntimeRoot => { + ("overlap-input", HlsOriginAccountStatus::Blocked) + } + InvalidAccount::RuntimeAlias => ("account-a", HlsOriginAccountStatus::Blocked), + } +} + +#[tokio::test] +async fn handover_updates_all_viewers_without_followup_media_requests() { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let proxy_id = session.read().await.proxy_session_id.clone(); + for (port, token) in [(12101, "viewer-a"), (12102, "viewer-b")] { + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &proxy_id, + token, + &test_fingerprint_with_addr(test_addr_with_port(port)), + token, + ) + .await; + } + let mut events = app.event_manager.get_event_channel(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .expect("replacement account publishes"); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "account-a"); + let rows = app.active_users.panel_streams().await; + assert_eq!(rows.len(), 2); + assert!( + rows.iter().all(|row| row.provider.as_ref() == "account-a"), + "all viewer rows must change immediately: {rows:?}" + ); + let mut updated = Vec::new(); + while let Ok(event) = events.try_recv() { + if let shared::model::EventMessage::ActiveUser(shared::model::ActiveUserConnectionChange::Updated(row)) = event + { + updated.push(row); + } + } + assert_eq!(updated.len(), 2); + assert!(updated.iter().all(|row| row.provider.as_ref() == "account-a")); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); +} + +#[tokio::test] +async fn invalid_root_alias_matrix_rebinds_within_input_and_preserves_session() { + for invalid in [ + InvalidAccount::ExpiredRoot, + InvalidAccount::ExpiredAlias, + InvalidAccount::RemovedAlias, + InvalidAccount::DisabledRoot, + InvalidAccount::DisabledAlias, + InvalidAccount::RuntimeRoot, + InvalidAccount::RuntimeAlias, + ] { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let (old, expected_status) = invalidate_account(&app, &mut input, invalid); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", old, now).await; + let (key, proxy_id) = { + let session = session.read().await; + (session.key.clone(), session.proxy_session_id.clone()) + }; + let old_binding = session.read().await.origin_account_binding.clone().unwrap(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &proxy_id, + "matrix-viewer", + &test_fingerprint(), + "matrix-viewer", + ) + .await; + assert_eq!(hls_origin_account_status(&app.hls_ctx(), &old_binding), expected_status, "{invalid:?}"); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + let session = session.read().await; + let binding = session.origin_account_binding.as_ref().unwrap(); + assert_eq!( + binding.account_name.as_ref(), + if old == "account-a" { "overlap-input" } else { "account-a" }, + "{invalid:?}" + ); + assert_eq!(binding.input_name, input.name); + assert_eq!(binding.generation, 1); + assert_eq!((&session.key, &session.proxy_session_id), (&key, &proxy_id)); + assert!(session.origin_account_manifest_pending.is_some()); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert!(session.origin_provider_session_headers.is_empty()); + assert!(!prepared.fetch_url.contains(if old == "account-a" { "account-user" } else { "root-user" })); + let expected_provider = binding.account_name.clone(); + drop(session); + assert!( + app.active_users.panel_streams().await.iter().all(|row| row.provider == expected_provider), + "{invalid:?}" + ); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + } +} + +#[tokio::test] +async fn detached_failed_rebind_obeys_two_second_backoff_and_source_boundary() { + let mut input = overlap_provider_input(); + input.aliases = None; + let mut other = single_hls_provider_input("unrelated-source"); + other.id = 9; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone()), Arc::new(other.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", "removed-account", now).await; + let id = session.read().await.proxy_session_id.clone(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone()), Arc::new(other.clone())]); + for timestamp in [now, now + 1_999] { + assert!(prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Segment, + HlsOriginWorkClass::Demand, + timestamp + ) + .await + .is_err()); + assert_eq!(session.read().await.origin_account_rebind.consecutive_rebind_failures, 1); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + } + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone()), Arc::new(other)]); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Segment, + HlsOriginWorkClass::Demand, + now + 2_000, + ) + .await + .unwrap(); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().account_name, input.name); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + let reused_input = single_hls_provider_input("reused-input-id"); + reload_inputs(&app, vec![Arc::new(reused_input.clone())]); + assert!(prepare_hls_origin_runtime( + &app, + &session, + &reused_input, + "http://account.example.com/index.m3u8", + "http://account.example.com/index.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now + 4_000 + ) + .await + .is_err()); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().input_name, input.name); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + reload_inputs(&app, Vec::new()); + assert!(prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/index.m3u8", + "http://root.example.com/index.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now + 4_000 + ) + .await + .is_err()); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().input_name, input.name); +} + +#[tokio::test] +async fn concurrent_prepare_publishes_one_replacement() { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..8 { + let app = Arc::clone(&app); + let input = input.clone(); + let session = Arc::clone(&session); + tasks.spawn(async move { + let id = session.read().await.proxy_session_id.clone(); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + }); + } + while let Some(result) = tasks.join_next().await { + result.unwrap(); + } + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 1); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +pub(super) fn enable_isolated_rebind_cache(app: &Arc, path: &std::path::Path) { + app.app_config.config.store(Arc::new(Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(HlsCacheConfigDto { + cache_path: Some(path.to_string_lossy().into_owned()), + ..HlsCacheConfigDto::default() + }), + ..ReverseProxyConfigDto::default() + })), + ..Config::default() + })); +} + +pub(super) struct RebindOrigin { + pub(super) base_url: String, + pub(super) requests: Arc>>, + task: tokio::task::JoinHandle<()>, + block_old_media: Arc, + old_media_started: Arc, + release_old_media: Arc, +} +impl Drop for RebindOrigin { + fn drop(&mut self) { self.task.abort(); } +} + +#[derive(Clone, Copy)] +enum RebindMediaLayout { + AccountPath, + SharedPath, +} + +async fn rebind_origin(extra_tags: &'static str, layout: RebindMediaLayout) -> RebindOrigin { + rebind_origin_with_window(extra_tags, layout, None, None).await +} + +pub(super) async fn resume_origin(head: Arc) -> RebindOrigin { + rebind_origin_with_window("", RebindMediaLayout::AccountPath, Some(head), None).await +} + +#[derive(Default)] +pub(super) struct ReducedRetryOriginControl { + pub(super) recovering: std::sync::atomic::AtomicBool, + pub(super) recovery_requests: std::sync::atomic::AtomicUsize, + pub(super) retry_started: tokio::sync::Notify, + pub(super) release_retry: tokio::sync::Notify, +} + +impl ReducedRetryOriginControl { + async fn manifest_body(&self) -> &'static [u8] { + if self.recovering.load(Ordering::Acquire) { + let n = self.recovery_requests.fetch_add(1, Ordering::AcqRel) + 1; + eprintln!("app recovery HTTP request={n} status=200"); + if n == 13 { + self.retry_started.notify_one(); + self.release_retry.notified().await; + } + if n <= 12 { + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/reduced-retry/conflict.m3u8" + )) + } else { + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/reduced-retry/repaired.m3u8" + )) + } + } else { + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/baseline.m3u8")) + } + } +} + +pub(super) async fn reduced_retry_origin(control: Arc) -> RebindOrigin { + rebind_origin_with_window("", RebindMediaLayout::AccountPath, None, Some(control)).await +} + +pub(super) fn resume_origin_media(path: &str) -> Vec { + // Valid TS, distinct from the enabled channel-unavailable asset. A null packet + // identifies the exact origin URI as well, preventing old cache hits from passing. + let mut media = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/user_connections_exhausted.ts")) + .to_vec(); + let mut marker = [0xff; 188]; + marker[..4].copy_from_slice(&[0x47, 0x1f, 0xff, 0x10]); + marker[4..4 + path.len()].copy_from_slice(path.as_bytes()); + media.extend_from_slice(&marker); + media +} + +async fn rebind_origin_with_window( + extra_tags: &'static str, + layout: RebindMediaLayout, + live_head: Option>, + recovery: Option>, +) -> RebindOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + let block_old_media = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let old_media_started = Arc::new(tokio::sync::Notify::new()); + let release_old_media = Arc::new(tokio::sync::Semaphore::new(0)); + let (block, started, release) = + (Arc::clone(&block_old_media), Arc::clone(&old_media_started), Arc::clone(&release_old_media)); + let task = tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let requests = Arc::clone(&recorded); + let live_head = live_head.clone(); + let recovery = recovery.clone(); + let (block, started, release) = (Arc::clone(&block), Arc::clone(&started), Arc::clone(&release)); + tokio::spawn(async move { + let mut buffer = [0; 4096]; + let count = socket.read(&mut buffer).await.unwrap(); + let request = String::from_utf8_lossy(&buffer[..count]).to_string(); + let path = request.lines().next().unwrap().split_whitespace().nth(1).unwrap().to_string(); + let alias_cookie = request.to_ascii_lowercase().contains("origin-account=alias"); + requests.lock().await.push(request); + if block.load(Ordering::Acquire) && path.contains("root-user") && path.contains(".ts") { + started.notify_one(); + release.acquire().await.unwrap().forget(); + } + let replacement = path.contains("account-user"); + let body = if let Some(control) = recovery { + // The playlists use virtual segment URIs; resume_origin_media supplies their TS bodies. + if path.ends_with("m3u8") { + control.manifest_body().await.to_vec() + } else { + resume_origin_media(&path) + } + } else if let Some(head) = live_head { + if path.ends_with("m3u8") { + super::manifest_entry_freshness::origin_playlist_with_duration(head.load(Ordering::Acquire), 11) + } else { + resume_origin_media(&path) + } + } else if path.ends_with("m3u8") { + let (head, count) = match (layout, replacement) { + (RebindMediaLayout::AccountPath, true) => (108, 8), + (RebindMediaLayout::SharedPath, true) => (102, 7), + (RebindMediaLayout::AccountPath | RebindMediaLayout::SharedPath, false) => (100, 8), + }; + let mut manifest = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + manifest.push_str(extra_tags); + for seq in head..head + count { + let uri = match layout { + RebindMediaLayout::AccountPath => format!("{seq}.ts"), + RebindMediaLayout::SharedPath => format!( + "/common/{seq}.ts?account={}", + if replacement { "account-user" } else { "root-user" } + ), + }; + writeln!(manifest, "#EXTINF:4,\n{uri}").unwrap(); + } + manifest.into_bytes() + } else if path == "/shared/key.bin" { + vec![if alias_cookie { 0x41 } else { 0x52 }; 16] + } else if path.contains(".ts") { + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/channel_unavailable.ts" + )) + .to_vec() + } else { + vec![0x47; 8192] + }; + let cookie = if replacement { "alias" } else { "root" }; + let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nSet-Cookie: origin-account={cookie}; Path=/\r\nConnection: close\r\n\r\n", body.len()); + socket.write_all(response.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + }); + } + }); + RebindOrigin { base_url, requests, task, block_old_media, old_media_started, release_old_media } +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn segment_and_background_work_fetch_new_manifest_and_media_without_client_manifest() { + for (background, cross_host, provider_scheme) in + [(false, false, false), (true, false, false), (false, true, false), (true, false, true)] + { + let origin = + rebind_origin("", if cross_host { RebindMediaLayout::SharedPath } else { RebindMediaLayout::AccountPath }) + .await; + let cache_dir = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.url.clone_from(&origin.base_url); + input.aliases.as_mut().unwrap()[0].url = + if cross_host { origin.base_url.replace("127.0.0.1", "localhost") } else { origin.base_url.clone() }; + if provider_scheme { + let provider = Arc::new(crate::model::ConfigProvider::from(&shared::model::ConfigProviderDto { + name: "rebind-mirror".into(), + urls: vec![origin.base_url.clone().into()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + input.url = "provider://rebind-mirror".to_string(); + input.aliases.as_mut().unwrap()[0].url.clone_from(&input.url); + input.provider_configs = Some(vec![provider]); + } + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(cache_dir.path(), 300)), + vec![Arc::new(input.clone())], + ); + enable_isolated_rebind_cache(&app, cache_dir.path()); + app.hls.proxy.update_config(&app.app_config).await; + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + let lease_uri = hls_proxy_uri(&app, &id.0, "000000.ts").await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", input.url); + let initial = prepare_hls_origin_runtime( + &app, + &session, + &input, + &url, + &url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + app.connection_manager.release_managed_provider_handle(initial.preacquired_origin_account_handle); + session.write().await.origin_account_manifest_pending = Some(0); + let runtime = session.read().await.origin_account_runtime.clone().unwrap(); + runtime.prepare(&session, HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand).await.unwrap(); + // Wait for initial prefetches and retain one cached object as identity/cache evidence. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if session.read().await.active_segment_fetches == 0 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + for seq in 0..8 { + let context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let file = HlsSegmentFile { proxy_seq: seq, extension: "ts".to_string() }; + let outcome = + app.hls.proxy.segment_worker_pool().demand_fetch_and_wait(context, &file, current_time_millis()).await; + assert_eq!(outcome, tuliprox_hls::api::SegmentDemandFetchOutcome::Ready); + } + session + .write() + .await + .render_and_store_manifest(current_time_millis()) + .expect("initial ready origin window is published"); + assert!(session.read().await.established_manifest_recovery_binding().is_some()); + let (key, cached, target_seq) = { + let mut s = session.write().await; + let cached = s + .segments + .iter() + .find_map(|(seq, entry)| matches!(entry.status, SegmentCacheStatus::Ready { .. }).then_some(*seq)); + let seq = *s.segments.keys().next_back().unwrap(); + let cache_key = s.segments[&seq].cache_key.clone(); + s.segments.get_mut(&seq).unwrap().status = SegmentCacheStatus::Discovered; + s.origin_provider_session_headers + .insert(header::COOKIE, HeaderValue::from_static("old-account=must-disappear")); + // Prior hard failure must not send recovery back to root credentials. + s.require_fresh_manifest_commit( + tuliprox_hls::api::HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + ); + s.origin_refresh.consecutive_failures = 1; + s.origin_refresh.next_fetch_allowed_at_ms = 0; + (s.key.clone(), cached, (seq, cache_key)) + }; + app.hls.proxy.segment_cache().delete(&target_seq.1).await.unwrap(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let boundary = origin.requests.lock().await.len(); + if background { + session.write().await.queue_segment_fetch_candidate( + target_seq.0, + tuliprox_hls::api::SegmentFetchPriority::Prefetch, + current_time_millis(), + ); + let context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + app.hls.proxy.segment_worker_pool().wake_scheduler(context, current_time_millis()).await; + } else { + let uri = lease_uri.replace("000000.ts", &format!("{:06}.ts", target_seq.0)); + let response = get_response(Arc::clone(&app), &uri, None).await; + assert!( + matches!(response.status(), StatusCode::OK | StatusCode::SERVICE_UNAVAILABLE), + "{}", + response.status() + ); + } + let replacement_ready = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let ready = session.read().await.segments.iter().find_map(|(seq, entry)| { + (matches!(entry.status, SegmentCacheStatus::Ready { .. }) + && entry + .origin_fetch_ref + .as_ref() + .is_some_and(|reference| reference.resolved_origin_url.contains("account-user"))) + .then_some(*seq) + }); + if let Some(seq) = ready { + break seq; + } + tokio::task::yield_now().await; + } + }) + .await; + let new_proxy_seq = match replacement_ready { + Ok(seq) => seq, + Err(error) => { + let s = session.read().await; + panic!("replacement media: background={background} host={cross_host} provider={provider_scheme}: {error}; pending={:?} refresh={:?} episode={:?} requests={:?}", s.origin_account_manifest_pending, s.origin_refresh, s.origin_control.acceptance_episode.as_ref().map(|episode| (&episode.state, episode.full_burst_completed)), origin.requests.lock().await.iter().map(|request| request.lines().next().unwrap()).collect::>()); + } + }; + let target = HlsSegmentFile { proxy_seq: new_proxy_seq, extension: "ts".to_string() }; + let context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let outcome = + app.hls.proxy.segment_worker_pool().demand_fetch_and_wait(context, &target, current_time_millis()).await; + assert_eq!( + outcome, + tuliprox_hls::api::SegmentDemandFetchOutcome::Ready, + "background={background} cross_host={cross_host} provider={provider_scheme}" + ); + let new_uri = lease_uri.replace("000000.ts", &format!("{new_proxy_seq:06}.ts")); + let response = get_response(Arc::clone(&app), &new_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_body(response).await.as_ref(), + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + ); + let s = session.read().await; + assert_eq!((&s.key, &s.proxy_session_id), (&key, &id)); + assert_eq!(s.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "account-a"); + assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, 1); + assert!(s.origin_account_manifest_pending.is_none()); + if let Some(cached) = cached { + if cached != target_seq.0 { + assert!(matches!(s.segments[&cached].status, SegmentCacheStatus::Ready { .. })); + } + } + drop(s); + let requests = origin.requests.lock().await; + let replacement = &requests[boundary..]; + assert!( + replacement.iter().any(|request| request.starts_with("GET /live/account-user/account-pass/12345.m3u8 ")), + "{replacement:?}" + ); + assert!( + replacement.iter().any(|request| request.lines().next().unwrap().contains("account-user") + && !request.lines().next().unwrap().contains("m3u8")), + "{replacement:?}" + ); + assert!( + replacement.iter().all(|request| !request.contains("root-user") && !request.contains("old-account=")), + "{replacement:?}" + ); + } +} + +#[tokio::test] +async fn independent_sessions_share_source_capacity_and_leave_other_source_unchanged() { + let mut input = overlap_provider_input(); + let mut alias = input.aliases.as_ref().unwrap()[0].clone(); + alias.id = 3; + alias.name = Arc::from("account-b"); + input.aliases.as_mut().unwrap().push(alias); + let mut other = single_hls_provider_input("other-source"); + other.id = 8; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone()), Arc::new(other.clone())]); + let now = current_time_millis(); + let first = create_bound_hls_test_session(&app, &input, "first", &input.name, now).await; + let second = create_bound_hls_test_session(&app, &input, "second", &input.name, now).await; + let unaffected = create_bound_hls_test_session(&app, &other, "third", &other.name, now).await; + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone()), Arc::new(other.clone())]); + let mut handles = Vec::new(); + for session in [&first, &second] { + let id = session.read().await.proxy_session_id.clone(); + handles.push( + prepare_hls_origin_runtime( + &app, + session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(), + ); + } + assert_ne!( + first.read().await.origin_account_binding.as_ref().unwrap().account_name, + second.read().await.origin_account_binding.as_ref().unwrap().account_name + ); + assert_eq!(app.active_provider.get_provider_connections_count(), 2); + let unaffected = unaffected.read().await; + assert_eq!(unaffected.origin_account_binding.as_ref().unwrap().account_name, other.name); + assert_eq!(unaffected.origin_account_binding.as_ref().unwrap().generation, 0); + drop(handles); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +#[tokio::test] +async fn healthy_full_account_keeps_binding_and_cancelled_prepare_leaks_no_slot() { + let input = single_hls_provider_input("healthy"); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let occupied = app + .active_provider + .acquire_exact_connection_with_lease_for_session( + &input.name, + &test_addr(), + false, + 0, + ConnectionKind::Normal, + Some(crate::api::model::PlaybackLeaseRef::new("foreign-owner", crate::model::PlaybackKind::LiveHls)), + ) + .unwrap(); + let id = session.read().await.proxy_session_id.clone(); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + assert!(prepared.preacquired_origin_account_handle.is_none()); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 0); + assert_eq!(session.read().await.origin_account_rebind.last_rebind_attempt_at_ms, None); + app.connection_manager.release_provider_handle(Some(occupied)); + let config = app.active_provider.find_provider_config(&input.name).unwrap(); + app.active_provider.observe_account(ProviderAccountObservation { + name: Arc::clone(&input.name), + identity: config.account_identity(), + status: Some(shared::model::ProxyUserStatus::Banned), + exp_date: None, + }); + let lock = Arc::clone(&session.read().await.origin_account_prepare_lock); + let guard = lock.lock().await; + let task_session = Arc::clone(&session); + let task_app = Arc::clone(&app); + let task = tokio::spawn(async move { + prepare_hls_origin_runtime( + &task_app, + &task_session, + &input, + "http://account.example.com/index.m3u8", + "http://account.example.com/index.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Segment, + HlsOriginWorkClass::Demand, + now, + ) + .await + }); + tokio::task::yield_now().await; + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + drop(guard); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +async fn resource_rebind_fixture( + extra_tags: &'static str, +) -> (tempfile::TempDir, RebindOrigin, Arc, ConfigInput, HlsSessionHandle) { + let origin = rebind_origin(extra_tags, RebindMediaLayout::AccountPath).await; + let dir = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.url.clone_from(&origin.base_url); + input.aliases.as_mut().unwrap()[0].url.clone_from(&origin.base_url); + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(dir.path(), 300)), + vec![Arc::new(input.clone())], + ); + enable_isolated_rebind_cache(&app, dir.path()); + app.hls.proxy.update_config(&app.app_config).await; + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + let _uri = hls_proxy_uri(&app, &id.0, "000000.ts").await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", input.url); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + &url, + &url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + drop(prepared); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + (dir, origin, app, input, session) +} + +#[tokio::test] +async fn map_worker_initiates_rebind_and_fetches_requalified_map() { + let (_dir, origin, app, _input, session) = resource_rebind_fixture("#EXT-X-MAP:URI=\"init.mp4\"\n").await; + let id = session.read().await.proxy_session_id.clone(); + let old_map = crate::api::model::MapEntry::new( + &id, + ProxyMapId(90), + OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: format!("{}/live/root-user/root-pass/old.mp4", origin.base_url), + byte_range: None, + }, + "mp4".to_string(), + ); + session.write().await.maps.insert(ProxyMapId(90), old_map); + let segment_context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let context = tuliprox_hls::api::MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::clone(app.hls.proxy.segment_cache()), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: segment_context.origin_io, + }; + app.hls.proxy.map_worker_pool().wake_scheduler(context, current_time_millis()).await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if session.read().await.maps.values().any(|map| { + map.origin_key.resolved_origin_uri.contains("account-user") + && matches!(map.status, MapCacheStatus::Ready { .. }) + }) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement manifest schedules its new MAP"); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 1); + let requests = origin.requests.lock().await; + assert!( + requests.iter().any(|request| request.starts_with("GET /live/account-user/account-pass/init.mp4 ")), + "{requests:?}" + ); + assert!( + requests.iter().all(|request| !request.contains("root-user") && !request.contains("old-account=")), + "{requests:?}" + ); +} + +#[tokio::test] +async fn transient_key_and_media_misses_rebind_before_reusing_any_old_url() { + for (kind, shared_uri) in [ + (TransientResourceKind::Key, false), + (TransientResourceKind::Segment, false), + (TransientResourceKind::Key, true), + ] { + let tags = if shared_uri { + "#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"/shared/key.bin\"\n" + } else { + "#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n" + }; + let (_dir, origin, app, _input, session) = resource_rebind_fixture(tags).await; + let now = current_time_millis(); + let resource = TransientResourceRef::new( + kind, + if shared_uri { + format!("{}/shared/key.bin", origin.base_url) + } else { + format!("{}/live/root-user/root-pass/old.bin", origin.base_url) + }, + &app.get_encrypt_secret(), + now, + 300_000, + Some("bin".to_string()), + ); + let id = session.read().await.proxy_session_id.clone(); + { + let mut s = session.write().await; + s.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + s.transient.upsert_resources([resource.clone()]); + s.origin_provider_session_headers + .insert(header::COOKIE, HeaderValue::from_static("old-account=must-disappear")); + } + let uri = hls_proxy_uri(&app, &id.0, &format!("r/{}.bin", resource.id.0)).await; + let response = get_response(Arc::clone(&app), &uri, None).await; + assert!( + response.status() == StatusCode::SERVICE_UNAVAILABLE || response.status() == StatusCode::NOT_FOUND, + "{}", + response.status() + ); + assert!(response.headers().contains_key(header::RETRY_AFTER) || response.status() == StatusCode::NOT_FOUND); + let resource_id = resource.id.clone(); + let new_resource = session + .read() + .await + .transient + .resources + .values() + .find(|resource| { + resource.kind == kind + && (resource.resolved_origin_uri.contains("account-user") + || (shared_uri && resource.id != resource_id)) + }) + .cloned() + .expect("new manifest publishes replacement resources"); + let extension = new_resource.file_ext_hint.as_deref().unwrap(); + let uri = hls_proxy_uri(&app, &id.0, &format!("r/{}.{extension}", new_resource.id.0)).await; + let response = get_response(Arc::clone(&app), &uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let expected: &[u8] = if kind == TransientResourceKind::Segment { + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + } else if shared_uri { + &[0x41; 16] + } else { + &[0x47; 8192] + }; + assert_eq!(response_body(response).await.as_ref(), expected); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 1); + let requests = origin.requests.lock().await; + assert!(requests.iter().any(|request| !request.lines().next().unwrap().contains("m3u8")), "{requests:?}"); + assert!( + requests.iter().all(|request| !request.contains("root-user") && !request.contains("old-account=")), + "{requests:?}" + ); + } +} + +#[tokio::test] +async fn abort_after_allocating_replacement_releases_its_handle() { + let mut input = overlap_provider_input(); + input.account_disabled = true; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let (allocated_tx, allocated_rx) = tokio::sync::oneshot::channel(); + let task_app = Arc::clone(&app); + let task = tokio::spawn(async move { + let id = session.read().await.proxy_session_id.clone(); + let prepared = prepare_hls_origin_runtime( + &task_app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(); + allocated_tx.send(()).unwrap(); + futures::future::pending::<()>().await; + drop(prepared); + }); + allocated_rx.await.unwrap(); + assert_eq!(app.active_provider.get_provider_connections_count(), 1); + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +#[tokio::test] +async fn late_origin_response_and_cleanup_cannot_overwrite_replacement() { + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let manifest = normal_manifest(&format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n{}/live/root-user/root-pass/100.ts\n", + origin.base_url + )); + session.write().await.apply_origin_manifest(&manifest).unwrap(); + origin.block_old_media.store(true, Ordering::Release); + let context = super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let worker = Arc::clone(app.hls.proxy.segment_worker_pool()); + let old_request = tokio::spawn(async move { + worker + .demand_fetch_and_wait( + context, + &HlsSegmentFile { proxy_seq: 0, extension: "ts".to_string() }, + current_time_millis(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), origin.old_media_started.notified()) + .await + .expect("old request reaches controlled barrier"); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input)]); + let runtime = session.read().await.origin_account_runtime.clone().unwrap(); + runtime.prepare(&session, HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand).await.unwrap(); + let replacement = session.read().await.origin_account_binding.clone().unwrap(); + let boundary = origin.requests.lock().await.len(); + origin.release_old_media.add_permits(1); + let _old_outcome = tokio::time::timeout(Duration::from_secs(5), old_request).await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let s = session.read().await; + if s.active_segment_fetches == 0 && s.activity.active_origin_work_count.load(Ordering::Acquire) == 0 { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let s = session.read().await; + let binding = s.origin_account_binding.as_ref().unwrap(); + assert_eq!((&binding.account_name, binding.generation), (&replacement.account_name, replacement.generation)); + assert!(binding.is_active()); + assert!(s.segments.iter().any(|(seq, entry)| *seq > 0 && matches!(entry.status, SegmentCacheStatus::Ready { .. }))); + assert!(!matches!(s.segments[&0].status, SegmentCacheStatus::Ready { .. })); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + assert!(origin.requests.lock().await[boundary..].iter().all(|request| !request.contains("root-user"))); +} + +struct BlockAccountBeforeAcquire { + delegate: Arc, + app: std::sync::Weak, + account: Arc, + calls: AtomicUsize, +} +impl std::fmt::Debug for BlockAccountBeforeAcquire { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlockAccountBeforeAcquire").finish_non_exhaustive() + } +} +impl tuliprox_hls::api::HlsOriginAccountRuntime for BlockAccountBeforeAcquire { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result< + tuliprox_hls::api::HlsPreparedOriginAccount, + tuliprox_hls::api::HlsBoundAccountAcquireErrorKind, + >, + > + Send + + 'a, + >, + > { + Box::pin(async move { + let prepared = self.delegate.prepare(session, work, class).await?; + if self.calls.fetch_add(1, Ordering::AcqRel) == 1 { + let app = self.app.upgrade().unwrap(); + let provider = app.active_provider.find_provider_config(&self.account).unwrap(); + app.active_provider.observe_account(ProviderAccountObservation { + name: Arc::clone(&self.account), + identity: provider.account_identity(), + status: Some(shared::model::ProxyUserStatus::Banned), + exp_date: None, + }); + } + Ok(prepared) + }) + } +} + +#[tokio::test] +async fn account_blocked_between_prepare_and_acquire_still_rebinds_from_media_work() { + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + let mut config = app.app_config.config.load_full().as_ref().clone(); + config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap().max_concurrent_segment_fetches_global = 1; + app.app_config.config.store(Arc::new(config)); + app.hls.proxy.update_config(&app.app_config).await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let delegate = session.read().await.origin_account_runtime.clone().unwrap(); + session.write().await.origin_account_runtime = Some(Arc::new(BlockAccountBeforeAcquire { + delegate, + app: Arc::downgrade(&app), + account: Arc::clone(&input.name), + calls: AtomicUsize::new(0), + })); + let manifest = normal_manifest(&format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n{}/live/root-user/root-pass/100.ts\n", + origin.base_url + )); + session.write().await.apply_origin_manifest(&manifest).unwrap(); + let context = super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let _old_outcome = app + .hls + .proxy + .segment_worker_pool() + .demand_fetch_and_wait( + context, + &HlsSegmentFile { proxy_seq: 0, extension: "ts".to_string() }, + current_time_millis(), + ) + .await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let s = session.read().await; + if s.origin_account_binding.as_ref().is_some_and(|binding| binding.account_name.as_ref() == "account-a") + && s.segments.values().any(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. })) + { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("confirmed invalidity at acquisition initiates replacement"); + let requests = origin.requests.lock().await; + assert!(requests + .iter() + .any(|request| request.contains("account-user") && request.lines().next().unwrap().contains(".ts"))); + assert!(requests.iter().all(|request| !request.contains("root-user"))); +} + +#[tokio::test] +async fn identical_key_uri_caches_distinct_account_bytes_and_keeps_published_old_key() { + let (_dir, origin, app, mut input, session) = + resource_rebind_fixture("#EXT-X-KEY:METHOD=AES-128,URI=\"/shared/key.bin\"\n").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let old_key = TransientResourceRef::new( + TransientResourceKind::Key, + format!("{}/shared/key.bin", origin.base_url), + &app.get_encrypt_secret(), + now, + 300_000, + Some("bin".to_string()), + ); + let id = { + let mut current = session.write().await; + current.transient.upsert_resources([old_key.clone()]); + current.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("origin-account=root")); + current.proxy_session_id.clone() + }; + let old_uri = hls_proxy_uri(&app, &id.0, &format!("r/{}.bin", old_key.id.0)).await; + let response = get_response(Arc::clone(&app), &old_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x52; 16]); + assert!(session + .read() + .await + .transient + .ready_key_object_valid_until_ms(&id, &old_key.id, "bin", current_time_millis()) + .is_some()); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input)]); + let runtime = session.read().await.origin_account_runtime.clone().unwrap(); + runtime.prepare(&session, HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand).await.unwrap(); + let new_key = session + .read() + .await + .transient + .resources + .values() + .find(|resource| { + resource.kind == TransientResourceKind::Key + && resource.id != old_key.id + && resource.resolved_origin_uri == old_key.resolved_origin_uri + }) + .cloned() + .expect("new account publishes a separate key identity for the same URI"); + let extension = new_key.file_ext_hint.as_deref().unwrap(); + let prefix = old_uri.rsplit_once("/r/").unwrap().0; + let new_uri = format!("{prefix}/r/{}.{extension}", new_key.id.0); + let lease_id = old_uri.split('/').nth(5).unwrap(); + super::support::publish_test_transient_resource_membership(&app, &id.0, lease_id, &format!("{old_uri}\n{new_uri}")) + .await; + let response = get_response(Arc::clone(&app), &new_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x41; 16]); + let request_count = + origin.requests.lock().await.iter().filter(|request| request.starts_with("GET /shared/key.bin ")).count(); + let response = get_response(Arc::clone(&app), &old_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x52; 16]); + assert_eq!( + origin.requests.lock().await.iter().filter(|request| request.starts_with("GET /shared/key.bin ")).count(), + request_count, + "old published key remains a cache hit" + ); +} + +#[tokio::test] +async fn manifest_request_built_after_a_handoff_uses_the_snapshot_entry_and_reprepares_it() { + use super::super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}; + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + session.write().await.replace_origin_account_binding(None); + let id = session.read().await.proxy_session_id.clone(); + let root_url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + &root_url, + &root_url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(); + // Preserve the exact result across a controlled asynchronous boundary. + let (continue_tx, continue_rx) = tokio::sync::oneshot::channel(); + let task_app = Arc::clone(&app); + let task_session = Arc::clone(&session); + let task = tokio::spawn(async move { + continue_rx.await.unwrap(); + let mut io = super::provider_admission::test_hls_origin_io_context(&task_app); + io.account_runtime = task_session.read().await.origin_account_runtime.clone(); + if let Some(handle) = prepared.preacquired_origin_account_handle { + io = io.with_managed_provider_handle(handle); + } + let request = hls_origin_refresh_request( + &task_app, + &task_session, + tuliprox_hls::api::HlsPreparedOriginAccount { + entry: tuliprox_hls::api::LiveHlsOriginEntry::parse(&prepared.fetch_url).unwrap(), + snapshot: prepared.snapshot, + }, + HlsOriginRefreshOptions { + headers: HeaderMap::new(), + origin_io: io, + manifest_commit_requirement: tuliprox_hls::api::HlsManifestCommitRequirement::CommittedManifestAllowed, + acceptance_directive: tuliprox_hls::api::HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + rewrite_secret: task_app.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + tuliprox_hls::api::trigger_origin_refresh_sync(request).await + }); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + drop( + prepare_hls_origin_runtime( + &app, + &session, + &input, + &root_url, + &root_url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(), + ); + continue_tx.send(()).unwrap(); + assert!(task.await.unwrap()); + let requests = origin.requests.lock().await; + assert!(requests.iter().any(|request| request.starts_with("GET /live/account-user/account-pass/12345.m3u8 "))); + assert!(requests.iter().all(|request| !request.contains("root-user")), "{requests:?}"); +} + +struct PausedOriginPrepare { + delegate: Arc, + prepared: tokio::sync::Notify, + release: tokio::sync::Semaphore, +} + +impl std::fmt::Debug for PausedOriginPrepare { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("PausedOriginPrepare").finish_non_exhaustive() + } +} + +impl tuliprox_hls::api::HlsOriginAccountRuntime for PausedOriginPrepare { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result< + tuliprox_hls::api::HlsPreparedOriginAccount, + tuliprox_hls::api::HlsBoundAccountAcquireErrorKind, + >, + > + Send + + 'a, + >, + > { + Box::pin(async move { + let prepared = self.delegate.prepare(session, work, class).await?; + self.prepared.notify_one(); + self.release.acquire().await.unwrap().forget(); + Ok(prepared) + }) + } +} + +#[tokio::test] +async fn runtime_prepare_result_superseded_before_refresh_start_cannot_authorize_its_old_entry() { + use super::super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}; + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let runtime = Arc::new(PausedOriginPrepare { + delegate: session.read().await.origin_account_runtime.clone().unwrap(), + prepared: tokio::sync::Notify::new(), + release: tokio::sync::Semaphore::new(0), + }); + let mut io = super::provider_admission::test_hls_origin_io_context(&app); + io.account_runtime = Some(runtime.clone()); + let entry = tuliprox_hls::api::LiveHlsOriginEntry::parse(&format!( + "{}/live/root-user/root-pass/12345.m3u8", + origin.base_url + )) + .unwrap(); + let request = hls_origin_refresh_request( + &app, + &session, + tuliprox_hls::api::HlsPreparedOriginAccount { + entry, + snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot::from_session(&*session.read().await), + }, + HlsOriginRefreshOptions { + headers: HeaderMap::new(), + origin_io: io, + manifest_commit_requirement: tuliprox_hls::api::HlsManifestCommitRequirement::CommittedManifestAllowed, + acceptance_directive: tuliprox_hls::api::HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + rewrite_secret: app.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + // Force a new prepare and pause its actual app-adapter result before the start CAS. + let mut request = request; + request.origin_io.as_mut().unwrap().prepared_account = None; + let refresh = tokio::spawn(tuliprox_hls::api::maybe_trigger_origin_refresh_with_outcome(request)); + tokio::time::timeout(Duration::from_secs(5), runtime.prepared.notified()).await.unwrap(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let id = session.read().await.proxy_session_id.clone(); + let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + drop( + prepare_hls_origin_runtime( + &app, + &session, + &input, + &url, + &url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(), + ); + runtime.release.add_permits(1); + assert_eq!(refresh.await.unwrap(), tuliprox_hls::api::HlsOriginRefreshTriggerOutcome::SessionUnavailable); + assert!(origin.requests.lock().await.is_empty(), "superseded runtime result sends no origin request"); + assert!(!session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn failed_rebind_does_not_publish_replacement_provider() { + let mut input = overlap_provider_input(); + input.aliases = None; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "viewer", + &test_fingerprint(), + "viewer", + ) + .await; + let before = app.active_users.panel_streams().await; + let mut events = app.event_manager.get_event_channel(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + assert!(prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now + ) + .await + .is_err()); + assert_eq!(app.active_users.panel_streams().await, before); + while let Ok(event) = events.try_recv() { + assert!(!matches!( + event, + shared::model::EventMessage::ActiveUser(shared::model::ActiveUserConnectionChange::Updated(_)) + )); + } +} + +#[tokio::test] +async fn older_handover_cannot_overwrite_newer_provider() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "viewer", + &test_fingerprint(), + "viewer", + ) + .await; + let original = session.read().await.origin_account_binding.clone().unwrap(); + let publish = |name: &'static str, generation| { + HlsOriginAccountBinding::rebound( + input.name.clone(), + Arc::from(name), + original.session_owner.clone(), + generation, + now, + ) + }; + let binding_b = publish("account-b", 1); + let snapshot_b = publish_hls_origin_account_with_status(&app, &session, |current| { + current.handoff_origin_account(binding_b); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + let ready = Arc::new(tokio::sync::Notify::new()); + let resume = Arc::new(tokio::sync::Notify::new()); + let old = { + let app = app.clone(); + let session = session.clone(); + let ready = ready.clone(); + let resume = resume.clone(); + tokio::spawn(async move { + ready.notify_one(); + resume.notified().await; + publish_hls_origin_account_with_status(&app, &session, |_| Ok(snapshot_b)).await + }) + }; + ready.notified().await; + let binding_c = publish("account-c", 2); + publish_hls_origin_account_with_status(&app, &session, |current| { + current.handoff_origin_account(binding_c); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + let mut events = app.event_manager.get_event_channel(); + resume.notify_one(); + assert!(old.await.unwrap().is_err()); + assert_eq!(app.active_users.panel_streams().await[0].provider.as_ref(), "account-c"); + assert!(events.try_recv().is_err()); +} + +#[tokio::test] +async fn stale_media_registration_cannot_restore_previous_provider() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "viewer", + &test_fingerprint(), + "viewer", + ) + .await; + let gate = session.read().await.origin_account_prepare_lock.clone(); + let held = gate.lock().await; + let fp = test_fingerprint(); + let headers = HeaderMap::new(); + let access = test_hls_access_context_with(id.clone(), HlsAccessLeaseId("viewer".into()), "viewer", fp.key.clone()); + let mut registration = + Box::pin(super::super::ensure_hls_cache_stream_registered(&app, &fp, &headers, &access, &session)); + assert!(futures::poll!(registration.as_mut()).is_pending()); + publish_hls_origin_account_with_status(&app, &session, |current| { + let old = current.origin_account_binding.as_ref().unwrap(); + let next = HlsOriginAccountBinding::rebound( + input.name.clone(), + Arc::from("account-b"), + old.session_owner.clone(), + old.generation + 1, + current_time_millis(), + ); + current.handoff_origin_account(next); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + drop(held); + assert_eq!(registration.await.unwrap().provider.as_ref(), "account-b"); + assert_eq!(app.active_users.panel_streams().await[0].provider.as_ref(), "account-b"); +} + +#[tokio::test] +async fn initiating_viewer_cancellation_does_not_lose_handover_updates() { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + for (port, token) in [(12201, "initiator"), (12202, "remaining")] { + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + token, + &test_fingerprint_with_addr(test_addr_with_port(port)), + token, + ) + .await; + } + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let published = Arc::new(tokio::sync::Notify::new()); + let request = { + let app = app.clone(); + let session = session.clone(); + let id = id.clone(); + let published = published.clone(); + tokio::spawn(async move { + let _prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + published.notify_one(); + std::future::pending::<()>().await; + }) + }; + published.notified().await; + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + assert!(app.active_users.panel_streams().await.iter().all(|row| row.provider.as_ref() == "account-a")); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "late-viewer", + &test_fingerprint_with_addr(test_addr_with_port(12203)), + "late-viewer", + ) + .await; + let rows = app.active_users.panel_streams().await; + assert_eq!(rows.len(), 3); + assert!(rows.iter().all(|row| row.provider.as_ref() == "account-a")); +} + +#[tokio::test] +async fn late_join_and_status_snapshot_use_current_origin_account() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "existing", + &test_fingerprint(), + "existing", + ) + .await; + publish_hls_origin_account_with_status(&app, &session, |current| { + let previous = current.origin_account_binding.as_ref().unwrap(); + let next = HlsOriginAccountBinding::rebound( + input.name.clone(), + Arc::from("account-b"), + previous.session_owner.clone(), + previous.generation + 1, + current_time_millis(), + ); + current.handoff_origin_account(next); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "late", + &test_fingerprint_with_addr(test_addr_with_port(12301)), + "late", + ) + .await; + let rows = app.active_users.panel_streams().await; + assert_eq!(rows.len(), 2); + assert!(rows.iter().all(|row| row.provider.as_ref() == "account-b")); +} + +#[tokio::test] +async fn handover_waits_for_viewer_projection_without_holding_hls_write_lock() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let shared_id = super::super::hls_cache_shared_stream_id(&session.read().await.proxy_session_id); + let held = app.active_users.lock_shared_stream_provider_projection(shared_id).await; + let mut publish = Box::pin(publish_hls_origin_account_with_status(&app, &session, |current| { + Ok(HlsOriginAccountSnapshot::from_session(current)) + })); + assert!(futures::poll!(publish.as_mut()).is_pending()); + assert!(session.try_write().is_ok(), "manager wait must never retain an HLS lock"); + drop(held); + publish.await.unwrap(); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs b/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs index 2ac34e07c..01164019e 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs @@ -491,7 +491,7 @@ async fn tokens_without_entry_kind_keep_bad_request_without_session() { let response = fixture.token_request(&client("10.0.0.1", 50_121), &token, HeaderMap::new()).await; assert_eq!(response.status(), StatusCode::BAD_REQUEST); } - assert!(fixture.origin.requests().is_empty()); + assert_eq!(fixture.origin.requests().len(), 0); } #[tokio::test] @@ -507,7 +507,7 @@ async fn shared_hls_cache_on_never_recreates_from_token() { ); let response = fixture.token_request(&client("10.0.0.1", 50_131), &token, HeaderMap::new()).await; assert_ne!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - assert!(fixture.origin.requests().is_empty()); + assert_eq!(fixture.origin.requests().len(), 0); assert!(fixture.app_state.active_users.get_and_update_user_session("hls-user", hint).await.is_none()); } diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs index 2650ff9bd..85bb46bfd 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs @@ -21,6 +21,9 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_hls_origin_io_context( reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, } } diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs index 80f0951cf..4d48a48fb 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs @@ -156,8 +156,8 @@ fn hls_soft_overlap_delay_scales_with_tuliprox_target_pressure() { #[tokio::test] async fn hls_account_overlap_reclaim_preempts_speculative_session() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); let winner = create_bound_hls_test_session(&app_state, &input, "winner", "account-a", 1_000).await; let loser = create_bound_hls_test_session(&app_state, &input, "loser", "account-a", 1_000).await; let winner_proxy_session_id = winner.read().await.proxy_session_id.clone(); @@ -175,7 +175,9 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() { } let loser_generation = loser.read().await.activity.origin_work_generation; - super::super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await; + winner.write().await.mark_authorized_media_access(10_000); + winner.write().await.activity.last_delivered_media_at_ms = Some(10_000); + drop(prepare_overlap_origin(&app_state, &input, &winner, 10_000).await); assert!(app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000).await); assert!(!app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000).await); @@ -191,8 +193,8 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() { #[tokio::test] async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); let displaced = create_bound_hls_test_session(&app_state, &input, "displaced", "account-a", 1_000).await; let promoted = create_bound_hls_test_session(&app_state, &input, "promoted", "account-a", 1_000).await; let displaced_proxy_session_id = displaced.read().await.proxy_session_id.clone(); @@ -210,7 +212,9 @@ async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { } let displaced_generation = displaced.read().await.activity.origin_work_generation; - super::super::promote_elapsed_hls_account_overlaps(&app_state, 20_001).await; + promoted.write().await.mark_authorized_media_access(20_001); + promoted.write().await.activity.last_delivered_media_at_ms = Some(20_001); + drop(prepare_overlap_origin(&app_state, &input, &promoted, 20_001).await); let displaced_binding_mode = displaced.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); assert!(matches!( @@ -262,8 +266,7 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work( .await .expect("interactive work should use soft-active overlap before grace"); - let binding = - prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared"); + let binding = new_session.read().await.origin_account_binding.clone().expect("speculative binding is published"); assert_eq!(binding.account_name, input.name); assert!(matches!( &binding.binding_mode, @@ -273,11 +276,15 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work( } if displaced_proxy_session_id == &old_proxy_session_id )); assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + prepared_origin + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .map(|handle| &handle.allocation), Some(super::super::ProviderAllocation::Available(_)) )); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -324,14 +331,16 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { .await .expect("normal HLS policy should preempt active soft HLS binding"); - let new_binding = prepared_origin - .origin_account_binding_to_store - .as_ref() - .expect("preempting session should receive active binding"); + let new_binding = + normal_session.read().await.origin_account_binding.clone().expect("preempting session receives active binding"); assert_eq!(new_binding.account_name, input.name); assert!(matches!(new_binding.binding_mode, HlsOriginAccountBindingMode::Active)); assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + prepared_origin + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .map(|handle| &handle.allocation), Some(super::super::ProviderAllocation::Available(_)) )); let soft_session = soft_session.read().await; @@ -345,7 +354,7 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { )); drop(soft_session); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -437,7 +446,6 @@ async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal let result = super::super::prepare_hls_origin_policy_preempt_runtime( &app_state, - &soft_session, &input, "http://account.example.com/live/account-user/account-pass/soft.m3u8", "http://account.example.com/live/account-user/account-pass/soft.m3u8", @@ -496,12 +504,18 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai .expect("interactive work can use grace when normal acquire and overlap fail"); assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + prepared_origin + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .map(|handle| &handle.allocation), Some(super::super::ProviderAllocation::GracePeriod(_)) )); assert_eq!( - prepared_origin - .origin_account_binding_to_store + session + .read() + .await + .origin_account_binding .as_ref() .expect("grace binding should still bind the selected account") .account_name @@ -509,7 +523,7 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai input.name.as_ref() ); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); app_state.connection_manager.release_provider_handle(Some(occupied)); } @@ -715,3 +729,40 @@ fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_ super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted ); } + +async fn prepare_overlap_origin( + app: &Arc, + input: &ConfigInput, + session: &tuliprox_hls::api::HlsSessionHandle, + now_ms: u64, +) -> super::super::session::PreparedHlsOriginRuntime { + let id = session.read().await.proxy_session_id.clone(); + super::super::prepare_hls_origin_runtime( + app, + session, + input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Manifest, + super::super::HlsOriginWorkClass::ManifestInteractive, + now_ms, + ) + .await + .unwrap() +} + +#[tokio::test] +async fn healthy_prepare_detaches_an_unprotected_expired_binding_before_reacquisition() { + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "expired-protection", "account-a", 1_000).await; + let generation = session.read().await.activity.origin_work_generation; + drop(prepare_overlap_origin(&app, &input, &session, 10_000).await); + let current = session.read().await; + assert!(current.activity.origin_work_generation > generation); + assert!(current.origin_account_binding.as_ref().unwrap().generation > 0); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs index e65886aa3..6cd8428ba 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs @@ -21,6 +21,33 @@ use http_body_util::BodyExt; use shared::model::{HlsStripMode, InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster}; use std::{sync::Arc, time::Duration}; +#[tokio::test] +async fn origin_resource_no_account_available_includes_retry_after() { + let app_state = test_app_state(); + let access_context = test_hls_access_context( + ProxySessionId("shared-hls-session".to_string()), + HlsAccessLeaseId("access-lease".to_string()), + ); + + for reason in [ + super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted, + super::super::HlsOriginRuntimeNoAccountReason::OriginBindingPreempted, + ] { + let response = super::super::hls_origin_runtime_resource_failure_response( + &app_state, + &access_context, + super::super::HlsOriginRuntimeAcquireError::NoAccountAvailable { reason }, + ); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{reason:?}"); + assert_eq!( + response.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), + Some("2"), + "{reason:?}" + ); + } +} + #[tokio::test] async fn transient_origin_binding_requires_runtime_prepare_for_missing_or_detached_account() { let input = single_hls_provider_input("known-account"); @@ -52,7 +79,7 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho } let old_generation = session.read().await.activity.origin_work_generation; - super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 4_001).await; + super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 6_001).await; { let session = session.read().await; @@ -77,17 +104,16 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho 0, super::super::HlsOriginWorkKind::Manifest, super::super::HlsOriginWorkClass::ManifestInteractive, - 4_100, + 6_100, ) .await .expect("authorized origin work can reacquire a detached binding"); - let binding = - prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller"); + let binding = session.read().await.origin_account_binding.clone().expect("new binding is published atomically"); assert_eq!(binding.account_name.as_ref(), "account-a"); assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -122,6 +148,70 @@ async fn hls_account_binding_without_media_activity_expires_after_startup_window assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); } +#[tokio::test] +async fn fresh_hls_binding_has_bounded_startup_protection_despite_old_media_delivery() { + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await; + { + let mut s = session.write().await; + s.target_duration = Some(11); + s.activity.last_authorized_media_at_ms = Some(70_000); + s.activity.last_delivered_media_at_ms = Some(70_000); + } + let proxy_id = session.read().await.proxy_session_id.clone(); + for now in [200_001, 204_999, 205_000] { + let prepared = super::super::prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Resource, + super::super::HlsOriginWorkClass::Demand, + now, + ) + .await + .unwrap(); + let s = session.read().await; + let binding = s.origin_account_binding.as_ref().unwrap(); + assert!(binding.is_active()); + assert_eq!(binding.pinned_at_ms, 200_000, "preparing the same binding cannot extend startup protection"); + assert_eq!(binding.generation, 0); + assert_eq!(s.activity.last_delivered_media_at_ms, Some(70_000)); + drop(s); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + } + super::super::detach_unprotected_hls_origin_account_bindings(&app, 205_001).await; + assert!( + session.read().await.origin_account_binding.as_ref().unwrap().is_detached(), + "no media progress: releasable after five seconds" + ); +} + +#[tokio::test] +async fn hls_binding_expired_startup_protects_running_origin_work() { + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await; + let generation = { + let mut s = session.write().await; + s.target_duration = Some(11); + s.activity.last_authorized_media_at_ms = Some(70_000); + s.activity.last_delivered_media_at_ms = Some(70_000); + s.start_origin_work() + }; + super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_000).await; + assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_active()); + assert!(session.write().await.finish_origin_work(generation)); + super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_001).await; + assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached()); +} + #[tokio::test] async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() { let app_state = test_app_state(); @@ -145,6 +235,7 @@ async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_d host_local_index: 123, }, proxy_seq: 123, + media_start_ms: 0, duration_ms: 4_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs index eeeb814e6..e33d8f0ad 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs @@ -95,7 +95,7 @@ async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues assert!(matches!(session.maps.get(&ProxyMapId(1)).expect("map remains").status, MapCacheStatus::Queued { .. })); } assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs new file mode 100644 index 000000000..c8cc4608e --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs @@ -0,0 +1,287 @@ +use super::{ + super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}, + manifest_entry_freshness::open_manifest_for_new_lease, + origin_account_rebind::{ + enable_isolated_rebind_cache, reduced_retry_origin, resume_origin_media, ReducedRetryOriginControl, + }, + *, +}; +use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + trigger_origin_refresh_sync, HlsManifestCommitRequirement, HlsOriginAccountSnapshot, HlsPreparedOriginAccount, +}; + +const WAIT: Duration = Duration::from_secs(15); + +async fn ready(app: &Arc, session: &HlsSessionHandle, sequences: &[u64]) { + tokio::time::timeout(WAIT, async { + loop { + let s = session.read().await; + if sequences.iter().all(|seq| { + s.segments.get(seq).is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. })) + }) && s.active_segment_fetches == 0 + && !s.origin_refresh.in_flight + && s.activity.active_origin_work_count.load(Ordering::Acquire) == 0 + { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("autonomous segment work and singleflight complete"); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +async fn deliver(app: &Arc, session: &HlsSessionHandle, manifest: &str, expected: &[u64]) { + assert!(!manifest.contains("/cvs/hls/")); + assert!(!manifest.contains("#EXT-X-ENDLIST")); + for sequence in expected { + let (proxy_seq, url) = { + let s = session.read().await; + let (seq, entry) = + s.segments.iter().find(|(_, entry)| entry.origin_key.host_local_sequence == *sequence).unwrap(); + (*seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone()) + }; + let uri = manifest + .lines() + .find(|line| line.ends_with(&format!("/{proxy_seq:06}.ts"))) + .expect("segment URI is published by Tuliprox"); + let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await; + assert_eq!(response.status(), StatusCode::OK); + let bytes = response_body(response).await; + let path = reqwest::Url::parse(&url).unwrap().path().to_string(); + assert_eq!(bytes.as_ref(), resume_origin_media(&path)); + eprintln!("app delivered resource={path} bytes={}", bytes.len()); + } +} + +#[expect(clippy::too_many_lines, reason = "Keep the ordered end-to-end recovery scenario and its assertions together.")] +async fn repaired_reduced_retry(mode: HlsStartupMode) { + let control = Arc::new(ReducedRetryOriginControl::default()); + let origin = reduced_retry_origin(Arc::clone(&control)).await; + let cache = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.name = Arc::from("account-a"); + input.priority = 0; + input.url.clone_from(&origin.base_url); + let b = &mut input.aliases.as_mut().unwrap()[0]; + b.name = Arc::from("account-b"); + b.priority = 10; + b.url.clone_from(&origin.base_url); + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)), + vec![Arc::new(input.clone())], + ); + let available_b = HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-b"), + &ProxySessionId("availability-check".into()), + current_time_millis(), + ); + assert_eq!( + tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b), + tuliprox_hls::api::HlsOriginAccountStatus::Known + ); + enable_isolated_rebind_cache(&app, cache.path()); + let mut config = app.app_config.config.load_full().as_ref().clone(); + let hls = config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap(); + hls.startup = (&HlsStartupConfigDto { mode, ..Default::default() }).into(); + hls.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast; + app.app_config.config.store(Arc::new(config)); + enable_channel_unavailable_custom_response(&app); + app.hls.proxy.update_config(&app.app_config).await; + create_active_hls_user_session(&app).await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + let lease = "reduced-retry-baseline"; + let _initial = open_manifest_for_new_lease(&app, &input, &url, lease).await; + let key = super::super::build_hls_origin_source(&input, "12345").session_key(); + let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap(); + ready(&app, &session, &[0, 1, 2, 3, 4, 5]).await; + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &HlsAccessLeaseId(lease.to_string()), + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + None, + HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .expect("complete baseline is published to the playback lease"); + assert_eq!(response.status(), StatusCode::OK); + let first = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + deliver(&app, &session, &first, &[1000]).await; + tokio::time::timeout(WAIT, async { + let proxy_id = session.read().await.proxy_session_id.clone(); + loop { + let snapshot = app + .hls + .proxy + .access_lease_response_snapshot(&HlsAccessLeaseId(lease.to_string()), &proxy_id, current_time_millis()) + .await + .unwrap(); + if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(0) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("baseline body completion is recorded before recovery pressure is evaluated"); + let (binding, baseline, commits, proxy_id) = { + let s = session.read().await; + assert!(s.activity.last_authorized_media_at_ms.is_some()); + assert!(s.activity.last_delivered_media_at_ms.is_some()); + assert!(s.established_manifest_recovery_binding().is_some()); + ( + s.origin_account_binding.clone().unwrap(), + s.origin_to_proxy.clone(), + s.origin_manifest_commits(), + s.proxy_session_id.clone(), + ) + }; + assert_eq!(binding.account_name.as_ref(), "account-a"); + eprintln!( + "{mode:?} baseline account={} binding_generation={} pinned_at_ms={} proxy_session={}", + binding.account_name, binding.generation, binding.pinned_at_ms, proxy_id.0 + ); + session.write().await.origin_control.path_condition = tuliprox_hls::HlsOriginPathCondition::HardFetchFailure; + let directive = + match tuliprox_hls::api::hls_manifest_acceptance_directive_for_session(&app.hls_ctx(), &session, &proxy_id) + .await + { + tuliprox_hls::api::HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive, + other => panic!("recovery pressure must evaluate: {other:?}"), + }; + eprintln!("app baseline directive={directive:?}"); + assert!(directive.recovery_pressure_guard.is_some(), "directive has current lease/session evidence"); + assert_eq!(directive.trigger, tuliprox_hls::HlsManifestAcceptanceTrigger::RecoveryRequired); + let mut io = super::provider_admission::test_hls_origin_io_context(&app); + io.account_runtime = session.read().await.origin_account_runtime.clone(); + let snapshot = HlsOriginAccountSnapshot::from_session(&*session.read().await); + let mut request = hls_origin_refresh_request( + &app, + &session, + HlsPreparedOriginAccount { entry: LiveHlsOriginEntry::parse(&url).unwrap(), snapshot }, + HlsOriginRefreshOptions { + headers: HeaderMap::new(), + origin_io: io, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + acceptance_directive: directive, + access_lease_id: Some(HlsAccessLeaseId(lease.to_string())), + rewrite_secret: app.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + request.now_ms = session.read().await.origin_refresh.next_fetch_allowed_at_ms.max(current_time_millis()); + control.recovering.store(true, Ordering::Release); + let refresh = tokio::spawn(trigger_origin_refresh_sync(request)); + tokio::time::timeout(WAIT, control.retry_started.notified()).await.expect("reduced HTTP attempt 13 starts"); + let episode_before = { + let s = session.read().await; + assert_eq!(s.origin_to_proxy, baseline); + assert_eq!(s.origin_manifest_commits(), commits, "first actual conflict commit did not mutate the timeline"); + let episode = s.origin_control.acceptance_episode.clone().unwrap(); + assert_eq!(episode.full_bursts_completed, 1); + assert!(episode.exhaustion_reason().is_some()); + assert!(episode.deterministic_conflict_receipt().is_none()); + eprintln!( + "{mode:?} before response 13: generation={} full_bursts={} reason={:?} timing={:?}", + episode.generation.0, + episode.full_bursts_completed, + episode.exhaustion_reason(), + episode.timing() + ); + episode + }; + let prepare_gate = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test(); + control.release_retry.notify_one(); + tokio::time::timeout(WAIT, prepare_gate.entered.notified()) + .await + .expect("repaired reduced retry commits and schedules media"); + let scheduled = { + let s = session.read().await; + let mut resources = s + .segment_prefetch_queue + .proxy_seqs() + .iter() + .map(|seq| { + let entry = &s.segments[seq]; + ( + *seq, + entry.origin_key.host_local_sequence, + entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone(), + ) + }) + .collect::>(); + resources.sort(); + assert_eq!(resources.iter().map(|(_, seq, _)| *seq).collect::>(), vec![1006, 1007, 1008]); + assert_eq!(s.origin_manifest_commits(), commits + 1); + assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation); + assert_eq!(s.origin_control.acceptance_generation, episode_before.generation); + assert!( + s.origin_control.acceptance_episode.is_none(), + "successful refresh bookkeeping consumes the completed episode" + ); + eprintln!("{mode:?} committed generation={} scheduled={resources:?}", episode_before.generation.0); + resources + }; + prepare_gate.prepare.notify_one(); + tokio::time::timeout(WAIT, prepare_gate.prepared.notified()).await.unwrap(); + prepare_gate.proceed.notify_one(); + let seqs = scheduled.iter().map(|(seq, _, _)| *seq).collect::>(); + ready(&app, &session, &seqs).await; + tokio::time::timeout(WAIT, refresh).await.unwrap().unwrap(); + assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13); + { + let requests = origin.requests.lock().await; + for (_, _, url) in &scheduled { + let path = reqwest::Url::parse(url).unwrap().path().to_string(); + let count = requests.iter().filter(|request| request.starts_with(&format!("GET {path} "))).count(); + assert!(count > 0, "queued resource was fetched before any client demand"); + eprintln!("{mode:?} autonomous origin GET {path} requests={count} status=Ready"); + } + assert!(requests.iter().all(|request| !request.contains("account-user")), "B remains unused"); + } + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &HlsAccessLeaseId(lease.to_string()), + HlsAccessLeaseState::Activated, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + None, + HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .expect("repaired live manifest is published"); + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + deliver(&app, &session, &body, &[1006, 1007, 1008]).await; + let s = session.read().await; + assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation); + assert_eq!(s.origin_account_binding.as_ref().unwrap().account_name, binding.account_name); + assert_eq!(s.origin_account_binding.as_ref().unwrap().pinned_at_ms, binding.pinned_at_ms); + assert_eq!(s.origin_account_binding.as_ref().unwrap().provider_binding_tag, binding.provider_binding_tag); + assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13); + eprintln!( + "{mode:?} complete: recovery_requests=13 account={} binding_generation={} origin_work=0 singleflight=false", + binding.account_name, binding.generation + ); + assert_eq!(s.proxy_session_id, proxy_id); + assert!(!s.origin_refresh.in_flight); + assert_eq!(s.active_segment_fetches, 0); + assert_eq!(s.activity.active_origin_work_count.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn hls_reduced_retry_delivers_repaired_origin_media_optimistic() { + repaired_reduced_retry(HlsStartupMode::Optimistic).await; +} + +#[tokio::test] +async fn hls_reduced_retry_delivers_repaired_origin_media_conservative() { + repaired_reduced_retry(HlsStartupMode::Conservative).await; +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs b/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs new file mode 100644 index 000000000..7cabb3ffc --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs @@ -0,0 +1,294 @@ +use super::{ + manifest_entry_freshness::open_manifest_for_new_lease, + origin_account_rebind::{enable_isolated_rebind_cache, resume_origin, resume_origin_media}, + *, +}; +use shared::model::{HlsStartupConfigDto, HlsStartupMode}; +use std::sync::atomic::AtomicU64; +use tuliprox_core::utils::current_time_millis; + +const PAUSE_MS: u64 = 130_000; +const WAIT: Duration = Duration::from_secs(5); + +async fn settle_origin(session: &HlsSessionHandle) { + tokio::time::timeout(WAIT, async { + loop { + let s = session.read().await; + if !s.origin_refresh.in_flight + && s.active_segment_fetches == 0 + && s.activity.active_origin_work_count.load(Ordering::Acquire) == 0 + { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("origin work settles before playback is stopped"); +} + +/// Model wall-clock absence without erasing actual media delivery, or relying on +/// Tokio's clock (account preparation and HTTP serving also read `SystemTime`). +async fn pause_playback(app: &Arc, session: &HlsSessionHandle, lease: &str) { + settle_origin(session).await; + assert!(app + .hls + .proxy + .segment_worker_pool() + .access_leases() + .write() + .await + .remove_access_lease(&HlsAccessLeaseId(lease.to_string())) + .is_some()); + app.active_users.release_session_streams_and_counted_reservation("hls-user", "hls-session-token").await; + let now = current_time_millis(); + let proxy_id = { + let mut s = session.write().await; + let age = |time: &mut Option| *time = time.map(|time| time.saturating_sub(PAUSE_MS)); + age(&mut s.activity.last_authorized_manifest_at_ms); + age(&mut s.activity.last_authorized_media_at_ms); + age(&mut s.activity.last_delivered_media_at_ms); + s.last_client_access_at_ms = s.last_client_access_at_ms.saturating_sub(PAUSE_MS); + let binding = s.origin_account_binding.as_mut().unwrap(); + binding.pinned_at_ms = binding.pinned_at_ms.saturating_sub(PAUSE_MS); + age(&mut binding.last_origin_io_at_ms); + age(&mut binding.last_reservation_refresh_at_ms); + age(&mut s.origin_refresh.last_fetch_started_at_ms); + age(&mut s.origin_refresh.last_fetch_finished_at_ms); + age(&mut s.origin_refresh.last_success_at_ms); + age(&mut s.origin_refresh.last_error_at_ms); + s.origin_refresh.next_fetch_allowed_at_ms = s.origin_refresh.next_fetch_allowed_at_ms.saturating_sub(PAUSE_MS); + if let Some(rendered) = s.last_rendered_manifest.as_mut() { + rendered.rendered_at_ms = rendered.rendered_at_ms.saturating_sub(PAUSE_MS); + rendered.valid_until_ms = rendered.valid_until_ms.saturating_sub(PAUSE_MS); + } + assert!(s.activity.last_delivered_media_at_ms.is_some_and(|last| now - last >= PAUSE_MS)); + assert!(!s.can_expire_idle_session(now, 300_000), "retained session is inside its 300-second timeout"); + s.proxy_session_id.clone() + }; + app.hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app.active_users, + &app.active_provider, + session, + &proxy_id, + now, + ) + .await; + super::super::detach_unprotected_hls_origin_account_bindings(app, now).await; + assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached()); + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "pause releases provider slots"); + for account in ["account-a", "account-b"] { + assert_eq!( + app.active_provider.provider_lease_usage(&Arc::from(account)).total(), + 0, + "no logical slot remains for {account}" + ); + } +} + +fn segment_uris(manifest: &str) -> Vec<&str> { + assert!(manifest.starts_with("#EXTM3U"), "live playlist: {manifest}"); + assert!(!manifest.contains("/cvs/hls/"), "replacement video must never satisfy resume: {manifest}"); + assert!(!manifest.contains("#EXT-X-ENDLIST"), "resume remains live"); + let uris = manifest.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::>(); + assert!(!uris.is_empty(), "live media playlist must contain media"); + uris +} + +async fn deliver_origin_segment(app: &Arc, session: &HlsSessionHandle, manifest: &str, head: u64) { + let uri = segment_uris(manifest)[0]; + let seq = uri.rsplit('/').next().unwrap().trim_end_matches(".ts").parse::().unwrap(); + let origin_url = { + let s = session.read().await; + let segment = &s.segments[&seq]; + assert!((head..head + 6).contains(&segment.origin_key.host_local_sequence), "current origin window"); + segment.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone() + }; + let path = reqwest::Url::parse(&origin_url).unwrap().path().to_string(); + let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = response_body(response).await; + assert_eq!(body.as_ref(), resume_origin_media(&path), "client receives the exact current origin object"); + let s = session.read().await; + assert!(s.activity.last_authorized_media_at_ms.is_some()); + assert!(s.activity.last_delivered_media_at_ms.is_some(), "fully consume the Tuliprox response before pausing"); + eprintln!( + "delivered origin={path} bytes={} authorized={:?} delivered={:?}", + body.len(), + s.activity.last_authorized_media_at_ms, + s.activity.last_delivered_media_at_ms + ); +} + +#[expect(clippy::too_many_lines, reason = "Keep the ordered pause/resume scenario and its assertions together.")] +async fn same_account_resume(mode: HlsStartupMode, resumes: usize) { + let head = Arc::new(AtomicU64::new(1113)); + let origin = resume_origin(Arc::clone(&head)).await; + let cache = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.name = Arc::from("account-a"); + input.priority = 0; + input.url.clone_from(&origin.base_url); + let b = &mut input.aliases.as_mut().unwrap()[0]; + b.name = Arc::from("account-b"); + b.priority = 10; + b.url.clone_from(&origin.base_url); + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)), + vec![Arc::new(input.clone())], + ); + let available_b = HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-b"), + &ProxySessionId("availability-check".into()), + current_time_millis(), + ); + assert_eq!( + tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b), + tuliprox_hls::api::HlsOriginAccountStatus::Known + ); + enable_isolated_rebind_cache(&app, cache.path()); + let mut config = app.app_config.config.load_full().as_ref().clone(); + config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap().startup = + (&HlsStartupConfigDto { mode, ..HlsStartupConfigDto::default() }).into(); + app.app_config.config.store(Arc::new(config)); + enable_channel_unavailable_custom_response(&app); + app.hls.proxy.update_config(&app.app_config).await; + assert_eq!(app.hls.proxy.session_idle_timeout_ms(), 300_000); + create_active_hls_user_session(&app).await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + let key = super::super::build_hls_origin_source(&input, "12345").session_key(); + let mut lease = "resume-initial".to_string(); + let first = open_manifest_for_new_lease(&app, &input, &url, &lease).await; + let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap(); + deliver_origin_segment(&app, &session, &first, 1113).await; + settle_origin(&session).await; + let proxy_id = session.read().await.proxy_session_id.clone(); + + for cycle in 0..resumes { + let old = session.read().await.origin_account_binding.clone().unwrap(); + assert_eq!(old.account_name.as_ref(), "account-a"); + pause_playback(&app, &session, &lease).await; + let current_head = 1126 + u64::try_from(cycle).unwrap() * 13; + head.store(current_head, Ordering::Release); + lease = format!("resume-{cycle}"); + create_active_hls_user_session(&app).await; + let boundary = origin.requests.lock().await.len(); + let started_at = current_time_millis(); + let barrier = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test(); + let (task_app, task_input, task_url, task_lease) = + (Arc::clone(&app), input.clone(), url.clone(), lease.clone()); + let opening = + tokio::spawn( + async move { open_manifest_for_new_lease(&task_app, &task_input, &task_url, &task_lease).await }, + ); + tokio::time::timeout(WAIT, barrier.entered.notified()).await.expect("manifest commit reaches scheduler"); + let retained = app.hls.proxy.sessions().get_by_key(&key).await.unwrap(); + assert!(Arc::ptr_eq(&session, &retained), "same shared session allocation"); + let (rebound, scheduled, delivered_before_prepare) = { + let s = session.read().await; + assert_eq!(s.key, key); + assert_eq!(s.proxy_session_id, proxy_id); + assert_eq!(s.target_duration, Some(11)); + let rebound = s.origin_account_binding.clone().unwrap(); + assert_eq!(rebound.account_name, old.account_name, "B is available; A is actually rebound"); + assert!(rebound.is_active()); + assert!(rebound.generation > old.generation); + assert!(rebound.pinned_at_ms >= started_at); + let mut scheduled = s + .segment_prefetch_queue + .proxy_seqs() + .into_iter() + .map(|seq| { + let entry = &s.segments[&seq]; + assert!((current_head..current_head + 6).contains(&entry.origin_key.host_local_sequence)); + (seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone()) + }) + .collect::>(); + scheduled.sort(); + assert!(!scheduled.is_empty(), "capture jobs immediately after commit, before preparation"); + eprintln!("{mode:?} cycle={cycle} shared={key:?} proxy={proxy_id:?} account={} binding={}->{} pinned={} scheduled={scheduled:?}", rebound.account_name, old.generation, rebound.generation, rebound.pinned_at_ms); + (rebound, scheduled, s.activity.last_delivered_media_at_ms) + }; + barrier.prepare.notify_one(); + tokio::time::timeout(WAIT, barrier.prepared.notified()).await.expect("resource preparation completes"); + eprintln!( + "requests through prepare={:?}", + origin.requests.lock().await[boundary..] + .iter() + .map(|request| request.lines().next().unwrap()) + .collect::>() + ); + { + let s = session.read().await; + eprintln!( + "after prepare: binding={:?} queue={:?} states={:?}", + s.origin_account_binding, + s.segment_prefetch_queue.proxy_seqs(), + scheduled.iter().map(|(seq, _)| (*seq, &s.segments[seq].status)).collect::>() + ); + assert!( + scheduled.iter().all(|(seq, _)| matches!( + s.segments[seq].status, + SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::Ready { .. } + )), + "manifest segment jobs were lost during resource preparation" + ); + let current = s.origin_account_binding.as_ref().unwrap(); + assert_eq!(current.generation, rebound.generation, "fresh binding stays stable into media work"); + assert_eq!(current.pinned_at_ms, rebound.pinned_at_ms); + assert_eq!( + s.activity.last_delivered_media_at_ms, delivered_before_prepare, + "manifest fetch is not media delivery" + ); + } + barrier.proceed.notify_one(); + // No demand fetch or test-side requeue: every original job must progress autonomously. + tokio::time::timeout(WAIT, async { + loop { + let s = session.read().await; + if scheduled.iter().all(|(seq, _)| matches!(s.segments[seq].status, SegmentCacheStatus::Ready { .. })) { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("all originally scheduled resources reach Ready"); + let manifest = tokio::time::timeout(WAIT, opening).await.unwrap().unwrap(); + let requests = origin.requests.lock().await[boundary..] + .iter() + .map(|request| request.lines().next().unwrap().to_string()) + .collect::>(); + eprintln!("{mode:?} cycle={cycle} origin requests={requests:?}"); + for (_, url) in &scheduled { + let path = reqwest::Url::parse(url).unwrap().path().to_string(); + assert!( + requests.iter().any(|line| line == &format!("GET {path} HTTP/1.1")), + "scheduled origin request: {path}" + ); + } + assert!(requests.iter().all(|line| !line.contains("account-user")), "B remains unused"); + deliver_origin_segment(&app, &session, &manifest, current_head).await; + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, rebound.generation); + } + pause_playback(&app, &session, &lease).await; + assert!(origin.requests.lock().await.iter().all(|request| !request.contains("account-user"))); + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "no provider slot leak after repeated pauses"); +} + +#[tokio::test] +async fn hls_same_account_resume_keeps_scheduled_segments() { + same_account_resume(HlsStartupMode::Optimistic, 2).await; +} + +#[tokio::test] +async fn hls_same_account_resume_keeps_scheduled_segments_conservative() { + same_account_resume(HlsStartupMode::Conservative, 2).await; +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs index b464e1b64..bc1488e9d 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs @@ -343,6 +343,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_segment_entry( host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX), }, proxy_seq, + media_start_ms: proxy_seq * 4_000, duration_ms: 4_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs b/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs index 7199acc23..9a7cc551f 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs @@ -94,6 +94,8 @@ pub(in crate::api::endpoints::hls_api::tests) fn store_normal_manifest_body_rang SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, ); entry.duration_ms = 10_000; + entry.media_start_ms = proxy_seq * 10_000; + session.next_media_start_ms = session.next_media_start_ms.max(entry.media_start_ms + entry.duration_ms); session.segments.insert(proxy_seq, entry); } session.advance_media_readiness_generation(); diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs index cf1b8a190..b0231d0f9 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs @@ -225,6 +225,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_runtime_policy_t HlsLeasePlaybackMode::Live => "live", HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", + HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response", HlsLeasePlaybackMode::Ended => "ended", }); panic!( diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs b/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs index e29da4016..73182959d 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs @@ -265,7 +265,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn hls_session_last_media_at pub(in crate::api::endpoints::hls_api::tests) async fn assert_no_hls_cache_stream_registered( app_state: &Arc, ) { - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } pub(in crate::api::endpoints::hls_api::tests) async fn access_lease_session_token( diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs index 5090b3914..e259a478b 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs @@ -230,11 +230,16 @@ pub(in crate::api::endpoints::hls_api::tests) async fn run_recovery_outage( let episode = session.origin_control.acceptance_episode.as_ref().expect("bounded acceptance evidence"); assert!(episode.full_burst_completed); assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - last_episode_generation = Some(episode.generation.0); + if let Some(generation) = last_episode_generation { + assert_eq!(episode.generation.0, generation, "unchanged failures share the initial full-burst receipt"); + } else { + last_episode_generation = Some(episode.generation.0); + } } - assert!( - fixture.origin.manifest_request_count().saturating_sub(requests_before) - >= plan.total_candidates().saturating_mul(3) + assert_eq!( + fixture.origin.manifest_request_count().saturating_sub(requests_before), + plan.total_candidates() + fixture.refresh.retry_policy.attempt_count().saturating_sub(1) + 2, + "one configured full burst, its bounded reduced retries, then one sample per due refresh" ); let uri = format!("/hls/shared/live/{}/{}/000003.ts", fixture.proxy_session_id.0, fixture.lease_id.0); let cached = get_response(Arc::clone(&fixture.app_state), &uri, None).await; @@ -794,9 +799,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; let transition_buffer_seq = fixture.base_manifest.last_proxy_seq.saturating_add(1); let commit_guard_seq = transition_buffer_seq.saturating_add(1); + let base_tail = session.segments.get(&fixture.base_manifest.last_proxy_seq).expect("live base tail"); + let mut media_start_ms = base_tail.media_start_ms.saturating_add(base_tail.duration_ms); for segment in session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) { + // Shortened test buffers must remain contiguous on the session media clock. + segment.media_start_ms = media_start_ms; if segment.proxy_seq == transition_buffer_seq { segment.duration_ms = 1_000; } else if segment.proxy_seq == commit_guard_seq { @@ -804,6 +813,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr } else { segment.status = SegmentCacheStatus::Expired; } + media_start_ms = media_start_ms.saturating_add(segment.duration_ms); } session.advance_media_readiness_generation(); } @@ -915,7 +925,11 @@ pub(in crate::api::endpoints::hls_api::tests) async fn commit_prepared_terminal_ .playback_cursor .first_segment_completed_at_ms .expect("measured lease playback start") - .saturating_add(fixture.base_manifest.playlist_duration_ms); + // Enter the acquisition window while the live base and its key are still usable. + .saturating_add( + fixture.base_manifest.playlist_duration_ms.saturating_sub(fixture.base_manifest.target_duration_ms), + ); + let first = commit_terminal_tail_if_lease_reserve_requires_cutover( &fixture.app_state.hls_ctx(), &fixture.session, @@ -1309,6 +1323,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn assert_publication_late_t assert!(body.contains("#EXT-X-ENDLIST")); assert!(body.contains("/terminal/")); } + HlsLeasePlaybackMode::TerminalCustomResponse(_) => { + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest utf8"); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!(body.contains("/cvs/hls/")); + assert!(!body.contains("/hls/shared/live/")); + } HlsLeasePlaybackMode::TerminalUnavailable { .. } => { assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } @@ -1352,6 +1373,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn regression_origin_refresh_reque transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id, disabled_headers: None, diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs index 2d7657594..7072a7945 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs @@ -86,7 +86,7 @@ async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path() } #[tokio::test] -async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() { +async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhausted_recovery() { let fixture = prepared_terminal_cutover_fixture().await; let directive = terminal_cutover_acceptance_directive(&fixture).await; let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await; @@ -96,7 +96,7 @@ async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() { } #[tokio::test] -async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_splice() { +async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splice_deadline() { let temp_dir = tempfile::tempdir().expect("tempdir"); let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); enable_channel_unavailable_custom_response(&app_state); @@ -147,16 +147,30 @@ async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_s let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.status(), StatusCode::OK); assert!(!response.headers().contains_key(header::LOCATION)); - assert!(response_body(response).await.is_empty()); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest"); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + assert!(body.contains("/cvs/hls/")); + assert!(!body.contains("#EXT-X-MAP")); + assert!(!body.contains("/hls/shared/live/")); let lease = app_state .hls .proxy .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) .await - .expect("failed-closed lease snapshot"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + .expect("late custom lease snapshot"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await; + assert_eq!(repeated.status(), StatusCode::OK); + assert_eq!(response_body(repeated).await, body.as_bytes()); + let repeated_lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("stable terminal custom lease"); + assert_eq!(repeated_lease.playback_mode, lease.playback_mode); let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0); let terminal_response = get_response(app_state, &terminal_uri, None).await; assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND); diff --git a/backend/app/src/api/endpoints/hls_terminal_response.rs b/backend/app/src/api/endpoints/hls_terminal_response.rs index 4080c2846..a53838a7e 100644 --- a/backend/app/src/api/endpoints/hls_terminal_response.rs +++ b/backend/app/src/api/endpoints/hls_terminal_response.rs @@ -44,6 +44,7 @@ pub(super) async fn hls_manifest_terminal_preflight( ) -> HlsManifestTerminalPreflight { match &lease.playback_mode { HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return HlsManifestTerminalPreflight::ServeCommittedPlayback, HlsLeasePlaybackMode::Live => {} @@ -183,6 +184,13 @@ pub(super) fn hls_terminal_playback_response( }; Some(hls_response(body.to_owned())) } + HlsLeasePlaybackMode::TerminalCustomResponse(plan) => { + Some(if plan.matches_route(proxy_session_id, access_lease_id) { + hls_response(plan.manifest_body().to_owned()) + } else { + StatusCode::NOT_FOUND.into_response() + }) + } HlsLeasePlaybackMode::TerminalUnavailable { .. } => Some(StatusCode::SERVICE_UNAVAILABLE.into_response()), HlsLeasePlaybackMode::Live => None, HlsLeasePlaybackMode::Ended => Some(StatusCode::NOT_FOUND.into_response()), diff --git a/backend/app/src/api/endpoints/v1_api_config.rs b/backend/app/src/api/endpoints/v1_api_config.rs index 9bc148e40..ac1fb201a 100644 --- a/backend/app/src/api/endpoints/v1_api_config.rs +++ b/backend/app/src/api/endpoints/v1_api_config.rs @@ -1186,11 +1186,11 @@ mod tests { assert_eq!(api_proxy.server.len(), 1); assert_eq!(api_proxy.server[0].name, "main"); - assert!(api_proxy.server[0].protocol.is_empty()); - assert!(api_proxy.server[0].host.is_empty()); + assert_eq!(api_proxy.server[0].protocol.len(), 0); + assert_eq!(api_proxy.server[0].host.len(), 0); assert!(api_proxy.server[0].port.is_none()); - assert!(api_proxy.server[0].timezone.is_empty()); - assert!(api_proxy.server[0].message.is_empty()); + assert_eq!(api_proxy.server[0].timezone.len(), 0); + assert_eq!(api_proxy.server[0].message.len(), 0); assert!(api_proxy.server[0].path.is_none()); assert_eq!(api_proxy.user.len(), 1); assert!(!api_proxy.use_user_db); @@ -1206,8 +1206,8 @@ mod tests { assert_eq!(api_proxy.server.len(), 1); assert_eq!(api_proxy.server[0].name, "main"); - assert!(api_proxy.server[0].host.is_empty()); - assert!(api_proxy.user.is_empty()); + assert_eq!(api_proxy.server[0].host.len(), 0); + assert_eq!(api_proxy.user.len(), 0); } #[test] @@ -1217,8 +1217,8 @@ mod tests { filter_api_proxy_by_permissions(&mut api_proxy, permissions); - assert!(api_proxy.server.is_empty()); - assert!(api_proxy.user.is_empty()); + assert_eq!(api_proxy.server.len(), 0); + assert_eq!(api_proxy.user.len(), 0); assert!(!api_proxy.use_user_db); assert_eq!(api_proxy.auth_error_status, 403); } @@ -1270,20 +1270,20 @@ mod tests { filter_app_config_by_permissions(&mut app_config, Some(permissions)); assert_eq!(app_config.config.storage_dir, None); - assert!(app_config.sources.inputs.is_empty()); + assert_eq!(app_config.sources.inputs.len(), 0); assert!(app_config.sources.provider.is_none()); assert!(app_config.sources.templates.is_none()); assert!(app_config.mappings.is_none()); assert!(app_config.templates.is_none()); assert_eq!(app_config.sources.sources.len(), 1); - assert!(app_config.sources.sources[0].inputs.is_empty()); + assert_eq!(app_config.sources.sources[0].inputs.len(), 0); assert_eq!(app_config.sources.sources[0].targets.len(), 1); let target = &app_config.sources.sources[0].targets[0]; assert_eq!(target.name, "Default"); assert_eq!(target.id, 1); assert_eq!(target.enabled, shared::model::ConfigTargetDto::default().enabled); - assert!(target.output.is_empty()); + assert_eq!(target.output.len(), 0); assert!(target.curation.is_none(), "identity-only access must not disclose discovery credentials"); assert!(target.filter.is_empty()); assert!(target.options.is_none()); @@ -1296,7 +1296,7 @@ mod tests { let api_proxy = app_config.api_proxy.expect("api proxy should remain present"); assert_eq!(api_proxy.server.len(), 1); assert_eq!(api_proxy.server[0].name, "main"); - assert!(api_proxy.server[0].host.is_empty()); + assert_eq!(api_proxy.server[0].host.len(), 0); assert_eq!(api_proxy.user.len(), 1); } diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs index 0bdc1c88c..57f7bf774 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs @@ -344,8 +344,8 @@ async fn manual_update_target_ids_preserve_named_legacy_and_empty_bulk_requests( assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]); } else { assert!(!queued.targets.enabled); - assert!(queued.targets.targets.is_empty()); - assert!(queued.targets.target_names.is_empty()); + assert_eq!(queued.targets.targets.len(), 0); + assert_eq!(queued.targets.target_names.len(), 0); } } } @@ -582,7 +582,7 @@ async fn m3u_update_quality_playlist_update_status_reads_newest_timestamp_from_c assert_eq!(status.inputs[0].clusters[2].timestamp, 202); assert_eq!(status.inputs[1].input_id, 8); assert_eq!(status.inputs[1].last_update, None); - assert!(status.inputs[1].clusters.is_empty()); + assert_eq!(status.inputs[1].clusters.len(), 0); assert_eq!(status.inputs[2].input_id, 9); assert_eq!(status.inputs[2].last_update, Some(404)); assert_eq!(status.inputs[2].clusters.len(), 1, "Only the actually persisted M3U cluster is exposed"); diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs index c53329ee0..a6ee09985 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs @@ -50,7 +50,7 @@ async fn playlist_update_status_reload_returns_active_bus_facts_without_persisti PlaylistUpdateState::Failure, ))); let Json(status) = super::super::playlist_update_status(State(app)).await.unwrap(); - assert!(status.active_updates.is_empty()); + assert_eq!(status.active_updates.len(), 0); assert_eq!( std::fs::read(path.join("status.json")).unwrap(), before, diff --git a/backend/app/src/api/endpoints/xtream_api/tests.rs b/backend/app/src/api/endpoints/xtream_api/tests.rs index 09e43d1ca..2c8ddd5c8 100644 --- a/backend/app/src/api/endpoints/xtream_api/tests.rs +++ b/backend/app/src/api/endpoints/xtream_api/tests.rs @@ -747,7 +747,7 @@ fn m3u_timeshift_resolver_returns_resolved_url_for_flussonic_template() { .expect("M3U input handled by bridge"); assert_eq!(resolved.url, "http://provider.example/channel/video-1704067200-3600.m3u8"); - assert!(!resolved.discriminator.is_empty()); + assert_ne!(resolved.discriminator.len(), 0); } #[test] diff --git a/backend/app/tests/hls_origin_account_handover.rs b/backend/app/tests/hls_origin_account_handover.rs new file mode 100644 index 000000000..d95001c08 --- /dev/null +++ b/backend/app/tests/hls_origin_account_handover.rs @@ -0,0 +1,353 @@ +//! Real-process regression: an origin account publication must reach every `WebUI` socket immediately. +use axum::{ + extract::{OriginalUri, State}, + response::IntoResponse, + Router, +}; +use futures::{SinkExt, StreamExt}; +use shared::model::{ActiveUserConnectionChange, ProtocolMessage, SourcesConfigDto, StreamInfo, PROTOCOL_VERSION}; +use std::{ + fmt::Write, + path::{Path, PathBuf}, + process::Stdio, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::Notify, +}; +use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; + +type Socket = WebSocketStream>; + +#[derive(Clone)] +struct Origin { + catalog: String, + manifests: Arc, + segments: Arc, + proceed: Arc, +} + +async fn origin(State(state): State, OriginalUri(uri): OriginalUri) -> axum::response::Response { + let path = uri.path(); + if path == "/__continue" { + state.proceed.notify_one(); + return "continued".into_response(); + } + if path.ends_with("catalog.m3u") { + return ([("content-type", "audio/x-mpegurl")], state.catalog).into_response(); + } + if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("m3u8")) { + state.manifests.fetch_add(1, Ordering::AcqRel); + return ([("content-type", "application/vnd.apple.mpegurl")], "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n100.ts\n#EXTINF:4,\n101.ts\n#EXTINF:4,\n102.ts\n#EXTINF:4,\n103.ts\n#EXTINF:4,\n104.ts\n#EXTINF:4,\n105.ts\n").into_response(); + } + if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("ts")) { + state.segments.fetch_add(1, Ordering::AcqRel); + return ( + [("content-type", "video/mp2t")], + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ) + .into_response(); + } + axum::http::StatusCode::NOT_FOUND.into_response() +} + +struct Fixture { + directory: tempfile::TempDir, + base: String, + source: PathBuf, + process: tokio::process::Child, + origins: Vec>, + b: Origin, + proceed: Arc, + control: String, +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = self.process.start_kill(); + for origin in &self.origins { + origin.abort(); + } + } +} + +impl Fixture { + async fn start() -> Self { + let a = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let b = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let a_addr = format!("localhost:{}", a.local_addr().unwrap().port()); + let b_addr = b.local_addr().unwrap(); + let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = reserved.local_addr().unwrap().port(); + drop(reserved); + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + for name in ["web", "storage", "backup"] { + std::fs::create_dir(root.join(name)).unwrap(); + } + let frontend = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend/dist"); + let web = if frontend.join("index.html").exists() { frontend } else { root.join("web") }; + let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: true\n user_ui_enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n stream:\n hls_session_ttl_secs: 120\n hls_cache:\n cache_path: '{}'\n", web.display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display()); + let source_text = format!("inputs:\n - name: account-a\n type: m3u\n url: http://{a_addr}/catalog.m3u\n max_connections: 8\n aliases:\n - name: account-b\n url: http://{b_addr}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [account-a]\n targets:\n - name: handover\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n"); + let mut users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Handover test\nuser:\n - target: handover\n credentials:\n"); + for user in ["alice", "bob", "late"] { + write!(users, " - username: {user}\n password: test-password\n proxy: reverse\n server: local\n status: Active\n").unwrap(); + } + let config_path = root.join("config.yml"); + let source = root.join("source.yml"); + let proxy = root.join("api-proxy.yml"); + std::fs::write(&config_path, config).unwrap(); + std::fs::write(&source, source_text).unwrap(); + std::fs::write(&proxy, users).unwrap(); + let proceed = Arc::new(Notify::new()); + let catalog = format!("#EXTM3U\n#EXTINF:-1 tvg-id=\"handover\" group-title=\"Test\",Handover Channel\nhttp://{a_addr}/live/index.m3u8\n"); + let state = |catalog: String| Origin { + catalog, + manifests: Arc::new(AtomicUsize::new(0)), + segments: Arc::new(AtomicUsize::new(0)), + proceed: proceed.clone(), + }; + let a_state = state(catalog.clone()); + let b_state = state(catalog.replace(&a_addr, &b_addr.to_string())); + let mut origins = Vec::new(); + for (listener, state) in [(a, a_state), (b, b_state.clone())] { + origins.push(tokio::spawn(async move { + axum::serve(listener, Router::new().fallback(origin).with_state(state)).await.unwrap(); + })); + } + let log = std::fs::File::create(root.join("sut.log")).unwrap(); + let process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox")) + .args(["-s", "-c"]) + .arg(config_path) + .arg("-i") + .arg(&source) + .arg("-a") + .arg(proxy) + .current_dir(root) + .stdout(Stdio::from(log.try_clone().unwrap())) + .stderr(Stdio::from(log)) + .kill_on_drop(true) + .spawn() + .unwrap(); + Self { + directory, + base: format!("http://127.0.0.1:{port}"), + source, + process, + origins, + b: b_state, + proceed, + control: format!("http://{a_addr}/__continue"), + } + } + + async fn playback(&mut self, client: &reqwest::Client, username: &str) -> String { + let url = format!("{}/m3u?username={username}&password=test-password", self.base); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + assert!( + self.process.try_wait().unwrap().is_none(), + "SUT exited: {}", + std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap() + ); + if let Ok(response) = client.get(&url).send().await { + if let Ok(body) = response.text().await { + if let Some(url) = body.lines().find(|line| line.starts_with("http")) { + return url.to_string(); + } + } + } + assert!( + tokio::time::Instant::now() < deadline, + "playlist unavailable: {}", + std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap() + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + async fn wait_for_browser_handover(&self, viewer_clients: &[reqwest::Client], viewer_media: &[reqwest::Url]) { + eprintln!( + "BROWSER_READY web={} control={} fixture={}", + self.base, + self.control, + self.directory.path().display() + ); + // Keep the two manual test players running while the browser loads. Stop all + // media requests before publishing B; the browser must then update from events alone. + let mut playback = tokio::time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + () = self.proceed.notified() => break, + _ = playback.tick() => { + for (client, media) in viewer_clients.iter().zip(viewer_media) { + let response = client.get(media.clone()).send().await.unwrap(); + assert!(response.status().is_success()); + response.bytes().await.unwrap(); + } + } + } + } + } +} + +async fn socket(base: &str) -> Socket { + let (mut socket, _) = connect_async(format!("{}/ws", base.replace("http://", "ws://"))).await.unwrap(); + socket.send(Message::Binary(ProtocolMessage::Version(PROTOCOL_VERSION).to_bytes().unwrap())).await.unwrap(); + assert!( + matches!(socket.next().await.unwrap().unwrap(), Message::Binary(bytes) if bytes.as_ref() == [PROTOCOL_VERSION]) + ); + socket +} + +async fn frame(socket: &mut Socket) -> ProtocolMessage { + loop { + let next = tokio::time::timeout(Duration::from_secs(15), socket.next()) + .await + .expect("WebUI frame arrives without polling") + .unwrap() + .unwrap(); + if let Message::Binary(bytes) = next { + return ProtocolMessage::from_bytes(bytes).unwrap(); + } + } +} + +async fn snapshot(socket: &mut Socket) -> Vec { + socket.send(Message::Binary(ProtocolMessage::StatusRequest(String::new()).to_bytes().unwrap())).await.unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if let ProtocolMessage::StatusResponse(status) = + tokio::time::timeout_at(deadline, frame(socket)).await.expect("full status snapshot arrives") + { + return status.active_user_streams; + } + } +} + +async fn changed_viewers(socket: &mut Socket, before: &[StreamInfo]) { + let mut seen = std::collections::HashSet::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while seen.len() < before.len() { + match tokio::time::timeout_at(deadline, frame(socket)).await.expect("all viewers receive Updated frames") { + ProtocolMessage::ActiveUserResponse(ActiveUserConnectionChange::Updated(row)) + if row.provider.as_ref() == "account-b" => + { + let mut expected = before + .iter() + .find(|old| old.uid == row.uid && old.addr == row.addr) + .expect("same viewer row") + .clone(); + expected.provider = Arc::from("account-b"); + assert_eq!(row, expected, "handover changes only the displayed provider"); + seen.insert(row.uid); + } + ProtocolMessage::StatusResponse(_) => panic!("handover must arrive as incremental Updated frames"), + _ => {} + } + } +} + +#[tokio::test] +async fn hls_origin_account_handover() { + let mut fixture = Fixture::start().await; + let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap(); + let mut canonical = Vec::new(); + let mut viewer_clients = Vec::new(); + let mut viewer_media = Vec::new(); + for viewer in ["alice", "bob"] { + let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap(); + let mut entry = fixture.playback(&client, viewer).await; + let mut manifest = None; + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while tokio::time::Instant::now() < deadline { + let response = client.get(&entry).send().await.unwrap(); + if response.status().is_success() { + let base = response.url().clone(); + let body = response.text().await.unwrap(); + let segments = body + .lines() + .filter(|line| !line.starts_with('#') && !line.is_empty()) + .map(|line| base.join(line).unwrap()) + .collect::>(); + if let Some(segment) = segments.first() { + if body.contains("#EXT-X-STREAM-INF:") { + entry = segment.to_string(); + continue; + } + let media = client.get(segment.clone()).send().await.unwrap(); + assert!(media.status().is_success()); + assert!( + !media.bytes().await.unwrap().is_empty(), + "viewer actually consumes media before the segment gap" + ); + viewer_media.push(segment.clone()); + manifest = Some(base); + break; + } + } + tokio::task::yield_now().await; + } + canonical.push(manifest.expect("viewer has a canonical playable manifest")); + viewer_clients.push(client); + } + let mut ui_a = socket(&fixture.base).await; + let mut ui_b = socket(&fixture.base).await; + let mut before = snapshot(&mut ui_a).await; + assert_eq!( + before.len(), + 2, + "canonical={canonical:?} SUT log: {}", + std::fs::read_to_string(fixture.directory.path().join("sut.log")).unwrap() + ); + assert!(before.iter().all(|row| row.provider.as_ref() == "account-a")); + let mut before_b = snapshot(&mut ui_b).await; + assert_eq!(before_b.len(), 2); + let manual = std::env::var_os("TULIPROX_HANDOVER_BROWSER_CHECK").is_some(); + if manual { + fixture.wait_for_browser_handover(&viewer_clients, &viewer_media).await; + // Playback above may advance timestamps; capture the exact pre-publication rows. + before = snapshot(&mut ui_a).await; + before_b = snapshot(&mut ui_b).await; + assert_eq!(before.len(), 2); + assert_eq!(before_b.len(), 2); + } + let mut sources: SourcesConfigDto = + serde_saphyr::from_str(&std::fs::read_to_string(&fixture.source).unwrap()).unwrap(); + sources.inputs[0].account_disabled = true; + let config = client.get(format!("{}/api/v1/config", fixture.base)).send().await.unwrap(); + let revision = + config.headers().get(shared::utils::HEADER_CONFIG_SOURCES_REVISION).expect("source revision").clone(); + let response = client + .post(format!("{}/api/v1/config/sources", fixture.base)) + .header(reqwest::header::IF_MATCH, revision) + .json(&sources) + .send() + .await + .unwrap(); + assert!(response.status().is_success(), "source reload: {}", response.text().await.unwrap()); + // Only a manifest refresh triggers the handover; neither viewer requests another segment or status. + let response = client.get(canonical[0].clone()).send().await.unwrap(); + assert!(response.status().is_success() || response.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE); + changed_viewers(&mut ui_a, &before).await; + changed_viewers(&mut ui_b, &before_b).await; + eprintln!( + "HANDOVER_EVENTS_DELIVERED replacement_manifest_requests={} replacement_segment_requests={}", + fixture.b.manifests.load(Ordering::Acquire), + fixture.b.segments.load(Ordering::Acquire) + ); + let after = snapshot(&mut ui_a).await; + assert_eq!(after.len(), 2); + assert!(after.iter().all(|row| row.provider.as_ref() == "account-b")); + if manual { + eprintln!("BROWSER_SWITCHED web={} control={}", fixture.base, fixture.control); + fixture.proceed.notified().await; + } + fixture.process.kill().await.unwrap(); +} diff --git a/backend/core/src/utils/network/content_coding.rs b/backend/core/src/utils/network/content_coding.rs index aca047ca9..3826a878f 100644 --- a/backend/core/src/utils/network/content_coding.rs +++ b/backend/core/src/utils/network/content_coding.rs @@ -793,7 +793,7 @@ mod tests { let response = local_response(StatusCode::OK, &[], encoded_binary.clone()).await; let mut untouched = decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await.expect("identity setup"); - assert!(untouched.decoded_from.is_empty()); + assert_eq!(untouched.decoded_from.len(), 0); assert_eq!(read_to_end_limited(&mut untouched.body, 1024).await.expect("untouched body"), encoded_binary); } @@ -806,7 +806,7 @@ mod tests { .await .expect("identity setup"); - assert!(decoded.decoded_from.is_empty()); + assert_eq!(decoded.decoded_from.len(), 0); assert_eq!(decoded.content_coding_observation(), None); assert!(!decoded.headers.contains_key(CONTENT_ENCODING)); assert_eq!(read_to_end_limited(&mut decoded.body, 1024).await.expect("identity body"), encoded); diff --git a/backend/core/src/utils/network/request.rs b/backend/core/src/utils/network/request.rs index d914568a6..e189356fc 100644 --- a/backend/core/src/utils/network/request.rs +++ b/backend/core/src/utils/network/request.rs @@ -52,11 +52,11 @@ pub enum ResourceDestination { } /// How long a verdict that allows a URL to be handed to a client is remembered. -const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_secs(60); +const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_mins(1); /// How long a verdict that keeps a destination away from clients is remembered. Both classes fail /// closed, so they can be cached longer than the one that decides exposure. -const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_secs(300); +const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_mins(5); /// How long a name that produced no answer within the resolution budget is remembered. /// diff --git a/backend/core/src/utils/network/request/tests/behavior.rs b/backend/core/src/utils/network/request/tests/behavior.rs index 2a529adf3..96516cd2b 100644 --- a/backend/core/src/utils/network/request/tests/behavior.rs +++ b/backend/core/src/utils/network/request/tests/behavior.rs @@ -108,7 +108,7 @@ async fn atomic_download_temp_files_are_unique_and_use_target_directory() { drop(second_output); drop(first_temp); drop(second_temp); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[test] @@ -181,7 +181,7 @@ async fn local_epg_file_respects_max_download_bytes() { assert!(err.to_string().contains("exceeds configured limit")); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -213,7 +213,7 @@ async fn file_url_epg_source_respects_streamed_size_limit() { assert!(err.to_string().contains("exceeds configured limit")); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -243,7 +243,7 @@ async fn local_epg_replace_error_cleans_temp_file() { .expect_err("replacing a directory should fail"); assert!(persist.is_dir()); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -273,7 +273,7 @@ async fn local_epg_read_error_preserves_existing_cache_and_cleans_temp_file() { .expect_err("reading a directory as an EPG file should fail"); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -301,7 +301,7 @@ async fn unsupported_epg_url_scheme_is_rejected_without_network_access() { assert!(err.to_string().contains("Unsupported EPG URL scheme 'ftp'")); assert!(!persist.exists()); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[cfg(windows)] @@ -370,7 +370,7 @@ async fn remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file() { assert!(err.to_string().contains("exceeds configured limit")); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); server_handle.abort(); } @@ -409,7 +409,7 @@ async fn remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file() .expect_err("truncated response body should fail"); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); server_handle.abort(); } @@ -466,7 +466,7 @@ async fn parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomic assert_eq!(accepted.load(Ordering::SeqCst), 2); assert_eq!(max_active.load(Ordering::SeqCst), 1); assert_eq!(tokio::fs::read(&persist).await.expect("read refreshed cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); server_handle.abort(); } diff --git a/backend/core/src/utils/network/request/tests/persistence.rs b/backend/core/src/utils/network/request/tests/persistence.rs index e4b1fa9a0..56032c4a1 100644 --- a/backend/core/src/utils/network/request/tests/persistence.rs +++ b/backend/core/src/utils/network/request/tests/persistence.rs @@ -37,5 +37,5 @@ async fn file_url_epg_source_is_copied_to_persist_path() { tokio::fs::read_to_string(&persist).await.expect("persisted content"), "BEGIN:VCALENDAR\nEND:VCALENDAR\n" ); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } diff --git a/backend/curation/src/coordinator/tests.rs b/backend/curation/src/coordinator/tests.rs index 0dd49fded..3f8e085ea 100644 --- a/backend/curation/src/coordinator/tests.rs +++ b/backend/curation/src/coordinator/tests.rs @@ -156,7 +156,7 @@ async fn empty_and_no_match_are_complete_but_disabled_sources_are_not_configured let CurationRunOutcome::Complete(result) = run(&config(None), &catalog(), &server).await else { panic!("valid empty or no-match is complete") }; - assert!(result.memberships.is_empty()); + assert_eq!(result.memberships.len(), 0); assert_eq!(result.selectors.len(), 2); } let server = TestServer::new(http_response(200, PAGE)).await; diff --git a/backend/curation/src/kernel.rs b/backend/curation/src/kernel.rs index 674e8eaf4..6c6bc8a27 100644 --- a/backend/curation/src/kernel.rs +++ b/backend/curation/src/kernel.rs @@ -699,7 +699,7 @@ mod tests { panic!("an empty source is complete") }; assert_eq!(reference_count, 0); - assert!(memberships.is_empty()); + assert_eq!(memberships.len(), 0); let no_match = vec![reference(CurationMediaKind::Movie, "Missing", None, Some(8), Some(1))]; let SelectorOutcome::Complete { reference_count, memberships, .. } = @@ -708,7 +708,7 @@ mod tests { panic!("a complete source with no local match is complete") }; assert_eq!(reference_count, 1); - assert!(memberships.is_empty()); + assert_eq!(memberships.len(), 0); let matched = vec![reference(CurationMediaKind::Movie, "Matched", None, Some(7), Some(1))]; let SelectorOutcome::Complete { reference_count, memberships, .. } = diff --git a/backend/curation/src/test_support.rs b/backend/curation/src/test_support.rs index e4d6b7cff..fca33e304 100644 --- a/backend/curation/src/test_support.rs +++ b/backend/curation/src/test_support.rs @@ -36,7 +36,7 @@ impl TestServer { } pub(crate) async fn serve(responses: Vec>, delay: Duration, send_headers: bool) -> Self { - assert!(!responses.is_empty()); + assert_ne!(responses.len(), 0); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}/", listener.local_addr().unwrap()); let requests = Arc::new(Mutex::new(Vec::new())); diff --git a/backend/curation/src/tmdb/client/tests.rs b/backend/curation/src/tmdb/client/tests.rs index db054bf11..6eeae2d0a 100644 --- a/backend/curation/src/tmdb/client/tests.rs +++ b/backend/curation/src/tmdb/client/tests.rs @@ -136,7 +136,7 @@ async fn tmdb_empty_is_valid_only_at_the_start_with_coherent_totals() { for pages in [0, 1] { let body = json!({"page":1,"total_pages":pages,"total_results":0,"results":[]}).to_string(); let server = TestServer::new(http_response(200, &body)).await; - assert!(client(&server).fetch(&selector(100)).await.unwrap().is_empty()); + assert_eq!(client(&server).fetch(&selector(100)).await.unwrap().len(), 0); assert_eq!(server.requests.lock().unwrap().len(), 1); } for body in [ @@ -222,7 +222,7 @@ async fn tmdb_all_routes_preserve_headers_with_no_query_token_or_details() { let mut selector = selector(100); selector.kind = kind; selector.time_window = window; - assert!(client.fetch(&selector).await.unwrap().is_empty()); + assert_eq!(client.fetch(&selector).await.unwrap().len(), 0); let requests = server.requests.lock().unwrap(); assert_eq!(requests.len(), 1); assert!( @@ -285,7 +285,7 @@ async fn tmdb_identity_chunked_and_gzip_exact_and_plus_one_count_decoded_bytes() let mut budget = client.batch_budget(); let result = client.trending(&selector(100), &mut budget).await; if length == BODY_LIMIT { - assert!(result.unwrap().is_empty()); + assert_eq!(result.unwrap().len(), 0); } else { assert_eq!(result.unwrap_err(), TmdbFailure::BodyLimit); } diff --git a/backend/curation/src/trakt/mod.rs b/backend/curation/src/trakt/mod.rs index 6d1a10b5a..32cd230dc 100644 --- a/backend/curation/src/trakt/mod.rs +++ b/backend/curation/src/trakt/mod.rs @@ -343,7 +343,7 @@ mod tests { assert_eq!(evaluation.selectors.len(), 1); assert_eq!(evaluation.selectors[0].reference_count, 0); assert_eq!(evaluation.selectors[0].membership_count, 0); - assert!(evaluation.memberships.is_empty()); + assert_eq!(evaluation.memberships.len(), 0); server.abort(); } diff --git a/backend/dvr/src/recording/recording_conflict.rs b/backend/dvr/src/recording/recording_conflict.rs index b26a5d05c..d6832b3f9 100644 --- a/backend/dvr/src/recording/recording_conflict.rs +++ b/backend/dvr/src/recording/recording_conflict.rs @@ -320,7 +320,7 @@ mod tests { None, ); assert_eq!(preview.severity, ConflictSeverity::NoKnownConflict); - assert!(preview.overlap_segments.is_empty()); + assert_eq!(preview.overlap_segments.len(), 0); } #[test] diff --git a/backend/dvr/src/recording/recording_deletion.rs b/backend/dvr/src/recording/recording_deletion.rs index 97af7265e..36079a56e 100644 --- a/backend/dvr/src/recording/recording_deletion.rs +++ b/backend/dvr/src/recording/recording_deletion.rs @@ -706,7 +706,7 @@ mod tests { let dir = TempDir::new().expect("tempdir"); let (queue, shared_file, target) = delete_one_of_two_entries_sharing(&dir).await; assert!(target.still_referenced, "Bob still holds this file"); - assert!(target.paths_to_unlink().is_empty()); + assert_eq!(target.paths_to_unlink().len(), 0); let unlinked = execute_deletion_target(&target).await.expect("execute"); assert_eq!(unlinked, None, "nothing was unlinked"); assert!(shared_file.exists(), "Bob's recording must survive Alice's deletion"); diff --git a/backend/dvr/src/recording/recording_queue/tests/policy.rs b/backend/dvr/src/recording/recording_queue/tests/policy.rs index feb57ac94..1787fad51 100644 --- a/backend/dvr/src/recording/recording_queue/tests/policy.rs +++ b/backend/dvr/src/recording/recording_queue/tests/policy.rs @@ -233,7 +233,7 @@ fn the_entry_behind_a_finished_transfer_attaches_instead_of_downloading_again() }; let promoted = promote_from_queue(&mut candidate); assert!(promoted.is_none(), "nothing needs to run"); - assert!(candidate.active.is_empty()); + assert_eq!(candidate.active.len(), 0); assert!(candidate.queue.is_empty(), "bob left the queue"); let bob = candidate.finished.iter().find(|task| task.uuid == "bob").expect("bob is filed"); assert_eq!(bob.state, RecordingTaskState::Completed); diff --git a/backend/dvr/src/recording/recording_retention.rs b/backend/dvr/src/recording/recording_retention.rs index 1049833fd..694f1d368 100644 --- a/backend/dvr/src/recording/recording_retention.rs +++ b/backend/dvr/src/recording/recording_retention.rs @@ -355,7 +355,7 @@ mod tests { let mut t = completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1); t.recording.completed_at = None; let out = compute_candidates(&[t], &config, 1_000_000_000); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -593,6 +593,6 @@ mod tests { let config = RetentionConfig::default(); let tasks = vec![completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1)]; let out = compute_candidates(&tasks, &config, 0); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } } diff --git a/backend/dvr/src/recording/recording_rule_scheduler.rs b/backend/dvr/src/recording/recording_rule_scheduler.rs index 90513581f..fc671c9c6 100644 --- a/backend/dvr/src/recording/recording_rule_scheduler.rs +++ b/backend/dvr/src/recording/recording_rule_scheduler.rs @@ -442,7 +442,7 @@ mod tests { let key = occurrence_key("r1", &source(), "ch-1", 1_000, "t:my show"); let tasks = vec![task("r1", &key)]; let out = plan_materializations(&rules, &programmes, &tasks, &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -450,7 +450,7 @@ mod tests { let rules = vec![new_episode_rule()]; let programmes = vec![epg_programme("My Show", -1, 100)]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -458,7 +458,7 @@ mod tests { let rules = vec![new_episode_rule()]; let programmes = vec![epg_programme("My Show", 5_000, 6_000)]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(2_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -468,7 +468,7 @@ mod tests { p.previously_shown = true; let programmes = vec![p]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -478,7 +478,7 @@ mod tests { let rules = vec![r]; let programmes = vec![epg_programme("My Show", 1_000, 2_000)]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -502,7 +502,7 @@ mod tests { // Set the EPG horizon to 1 hour — the next Sunday is // days away. let out = plan_materializations(&rules, &[], &[], &TombstoneSet::default(), now, Some(3_600)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -514,6 +514,6 @@ mod tests { let out = plan_materializations(&[rule], &[], &[], &TombstoneSet::default(), 0, None); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } } diff --git a/backend/dvr/src/recording/recording_worker.rs b/backend/dvr/src/recording/recording_worker.rs index 6a5eba473..9e3c909e6 100644 --- a/backend/dvr/src/recording/recording_worker.rs +++ b/backend/dvr/src/recording/recording_worker.rs @@ -405,7 +405,7 @@ pub async fn recovery_decision_for(final_path: &Path, partial: &Path) -> Recover #[cfg(test)] mod tests { - #[cfg(unix)] + #[cfg(target_os = "linux")] use super::spawn_retrying_busy_executable; use super::{ build_recording_args, classify_ffmpeg_failure, recording_partial_path, @@ -778,7 +778,8 @@ mod tests { let _ = fs::remove_dir_all(&recording.file_dir); } - #[cfg(unix)] + // Linux rejects execution while a writer is open; macOS permits it. + #[cfg(target_os = "linux")] #[tokio::test] async fn spawn_retries_while_executable_is_open_for_writing() { let script = fake_ffmpeg_script("busy", "#!/bin/sh\nexit 0\n"); @@ -970,8 +971,7 @@ mod tests { }; assert!(generated.status.success(), "{}", String::from_utf8_lossy(&generated.stderr)); let bytes = tokio::fs::read(&input).await?; - // This address is outside the usual environment's localhost exclusion. - let listener = tokio::net::TcpListener::bind("127.0.0.2:0").await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let url = reqwest::Url::parse(&format!("http://{}/api/v1/playlist/recording/secret/live", listener.local_addr()?))?; let (request_tx, request_rx) = tokio::sync::oneshot::channel(); @@ -991,7 +991,12 @@ mod tests { Ok::<_, std::io::Error>(()) }); let wrapper = directory.path().join("ffmpeg-with-proxy"); - tokio::fs::write(&wrapper, "#!/bin/sh\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n").await?; + // Explicitly remove proxy exclusions so localhost still exercises the bypass. + tokio::fs::write( + &wrapper, + "#!/bin/sh\nunset no_proxy NO_PROXY\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n", + ) + .await?; fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755))?; let mut recording = make_recording(chrono::Utc::now().timestamp(), 60); recording.url = url; diff --git a/backend/dvr/src/recording/recording_ws.rs b/backend/dvr/src/recording/recording_ws.rs index 862f59bec..9375f1ee2 100644 --- a/backend/dvr/src/recording/recording_ws.rs +++ b/backend/dvr/src/recording/recording_ws.rs @@ -274,7 +274,7 @@ mod tests { let queue = RecordingQueue::new(); let snapshot = recording_snapshot(&queue, &no_subject_claims(), &bare_app_config()).await; assert_eq!(snapshot.revision.0, 0); - assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.tasks.len(), 0); assert_eq!(snapshot.quota, RecordingQuotaSummaryDto::default(), "a session with no subject is charged nothing"); } @@ -283,7 +283,7 @@ mod tests { let queue = RecordingQueue::new(); let snapshot = recording_snapshot(&queue, &no_read_claims(), &bare_app_config()).await; assert_eq!(snapshot.revision.0, 0); - assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.tasks.len(), 0); } /// Every internal fact a recording carries that a client must never diff --git a/backend/hls/src/availability/manifest_failure.rs b/backend/hls/src/availability/manifest_failure.rs new file mode 100644 index 000000000..8c969ea10 --- /dev/null +++ b/backend/hls/src/availability/manifest_failure.rs @@ -0,0 +1,46 @@ +//! Publish degraded origin evidence before the manifest retry loop starts waiting. + +use super::reevaluation::register_terminal_reserve_reevaluation; +use crate::{ + manifest_fetch::{HlsOriginManifestFetchContext, OriginManifestFetchError}, + origin_progress::HlsOriginPathCondition, + post_refresh_availability::commit_post_refresh_terminal_fallback, + refresh::{HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason}, +}; +use std::sync::Arc; + +pub(crate) async fn record_retryable_manifest_attempt_failure( + context: &HlsOriginManifestFetchContext, + expected_progress_generation: u64, + error: &OriginManifestFetchError, + now_ms: u64, +) { + let action = { + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() + || session.origin_control.progress_generation != expected_progress_generation + { + return; + } + if matches!(error, OriginManifestFetchError::RetryableStatus(..)) { + session.origin_control.record_origin_response(now_ms); + } + // An individual soft failure must not weaken an already established hard failure or conflict. + if matches!( + session.origin_control.path_condition, + HlsOriginPathCondition::ProgressExpected | HlsOriginPathCondition::RetryableFetchFailure + ) { + session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure; + } + HlsPostRefreshAvailabilityAction::Reevaluate { + reason: HlsPostRefreshAvailabilityReason::RetryableManifestFailure, + origin_progress_generation: session.origin_control.progress_generation, + media_readiness_generation: session.activity.media_readiness_generation, + } + }; + let Some(ctx) = context.availability_runtime.as_ref().and_then(crate::hls_ctx::WeakHlsCtx::upgrade) else { + return; + }; + let registration = register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(&context.session)).await; + let _ = commit_post_refresh_terminal_fallback(ctx, Arc::clone(&context.session), action, registration).await; +} diff --git a/backend/hls/src/availability/mod.rs b/backend/hls/src/availability/mod.rs index fdfdc49f5..8e3ddc166 100644 --- a/backend/hls/src/availability/mod.rs +++ b/backend/hls/src/availability/mod.rs @@ -17,11 +17,18 @@ use super::{ HlsLiveReserveDeadline, HlsOriginProgressDecision, ProxySessionId, }; +mod manifest_failure; mod recovery_pressure; mod reevaluation; mod terminal_cutover; mod terminal_pending_owner; -pub(crate) use self::terminal_cutover::commit_terminal_tail_if_lease_reserve_requires_cutover_detailed; +pub(crate) use self::{ + manifest_failure::record_retryable_manifest_attempt_failure, + terminal_cutover::{ + commit_terminal_tail_if_lease_reserve_requires_cutover_detailed, terminal_bundle_failure_compatibility, + terminal_bundle_incompatibility, + }, +}; use self::{ recovery_pressure::{ evaluate_and_commit_session_recovery_pressure_in_snapshot, hls_recovery_pressure_policy, diff --git a/backend/hls/src/availability/reevaluation.rs b/backend/hls/src/availability/reevaluation.rs index 177ff7e05..b8028a006 100644 --- a/backend/hls/src/availability/reevaluation.rs +++ b/backend/hls/src/availability/reevaluation.rs @@ -347,7 +347,7 @@ async fn evaluate_post_refresh_terminal_attempt( cycle: &mut HlsAvailabilityReevaluationCycle, now_ms: u64, ) -> HlsTerminalReevaluationDecision { - if !matches!(mode, HlsAvailabilityReevaluationMode::PostRefresh(_)) { + if matches!(mode, HlsAvailabilityReevaluationMode::RecoveryPressure) { return HlsTerminalReevaluationDecision::Continue { terminal_evaluation_pending: false }; } match evaluate_active_terminal_leases_for_reevaluation(ctx, session, &owner_key.proxy_session_id, now_ms).await { @@ -439,7 +439,7 @@ async fn run_hls_availability_reevaluation_attempt( session: &mut HlsSessionHandle, owner_key: &mut HlsAvailabilityReevaluationOwnerKey, ownership: &HlsAvailabilityReevaluationOwnership, - refresh_request: &mut OriginRefreshRequest, + refresh_request: &mut Option, cycle: &mut HlsAvailabilityReevaluationCycle, ) -> HlsAvailabilityCycleDecision { if !ownership.is_current(owner_key) { @@ -463,6 +463,9 @@ async fn run_hls_availability_reevaluation_attempt( HlsTerminalReevaluationDecision::RestartCycle => return HlsAvailabilityCycleDecision::RestartCycle, HlsTerminalReevaluationDecision::Stop => return HlsAvailabilityCycleDecision::Stop, }; + let Some(refresh_request) = refresh_request.as_mut() else { + return HlsAvailabilityCycleDecision::FinishCycle; + }; let schedule = match evaluate_availability_refresh_attempt( ctx, session, @@ -482,10 +485,13 @@ async fn run_hls_availability_reevaluation_attempt( return HlsAvailabilityCycleDecision::FinishCycle; } let retry_from_ms = ctx.hls_proxy.now_ms(); - let Some(retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else { + let Some(mut retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else { warn!("HLS availability reevaluation cycle stopped: reason=deadline_elapsed"); return HlsAvailabilityCycleDecision::FinishCycle; }; + if let Some(deadline) = cycle.live_reserve_deadline { + retry_at_ms = retry_at_ms.min(deadline.next_reevaluation_at_ms); + } match wait_for_availability_owner_signal(ownership, retry_at_ms, retry_from_ms).await { HlsAvailabilityOwnerWaitOutcome::Cancelled => { ownership.discard_superseded(owner_key); @@ -504,7 +510,7 @@ async fn run_hls_availability_reevaluation( mut session: HlsSessionHandle, mut owner_key: HlsAvailabilityReevaluationOwnerKey, ownership: HlsAvailabilityReevaluationOwnership, - mut refresh_request: OriginRefreshRequest, + mut refresh_request: Option, ) { 'cycles: loop { let mut cycle = HlsAvailabilityReevaluationCycle::new(ctx.hls_proxy.now_ms()); @@ -605,6 +611,24 @@ pub(super) fn register_hls_availability_reevaluation_with_mode( let coordinator = ctx.hls_proxy.availability_reevaluations(); let registration_key = owner_key.clone(); coordinator.register(registration_key, mode, move |ownership| { - run_hls_availability_reevaluation(ctx, session, owner_key, ownership, refresh_request) + run_hls_availability_reevaluation(ctx, session, owner_key, ownership, Some(refresh_request)) + }) +} + +pub(super) fn register_terminal_reserve_reevaluation( + ctx: HlsCtx, + session: HlsSessionHandle, +) -> futures::future::BoxFuture<'static, HlsAvailabilityReevaluationRegistration> { + Box::pin(async move { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let Some(owner_key) = ctx.hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await + else { + return HlsAvailabilityReevaluationRegistration::Superseded; + }; + ctx.hls_proxy.availability_reevaluations().register( + owner_key.clone(), + HlsAvailabilityReevaluationMode::TerminalReserve, + move |ownership| run_hls_availability_reevaluation(ctx, session, owner_key, ownership, None), + ) }) } diff --git a/backend/hls/src/availability/terminal_cutover.rs b/backend/hls/src/availability/terminal_cutover.rs index b3f5edc7d..f21e9d92c 100644 --- a/backend/hls/src/availability/terminal_cutover.rs +++ b/backend/hls/src/availability/terminal_cutover.rs @@ -8,9 +8,9 @@ use super::{ recovery_pressure::{hls_recovery_pressure_policy, recovery_trigger_budget}, terminal_pending_owner::{ - commit_prepared_terminal_unavailable_after_owner_failure, commit_terminal_unavailable, - register_ready_terminal_pending_owner, register_terminal_pending_owner, terminal_asset_revision_guard, - terminal_resolution_with_failed_closed_fallback, + commit_late_custom_response, commit_prepared_terminal_unavailable_after_owner_failure, + commit_terminal_unavailable, register_ready_terminal_pending_owner, register_terminal_pending_owner, + terminal_asset_revision_guard, terminal_resolution_with_failed_closed_fallback, }, HlsDetailedTerminalResolution, HlsTerminalDecisionPurpose, HlsTerminalFailedClosedReason, HlsTerminalResolution, HLS_PLAYBACK_RATE_GUARD_MILLI, @@ -184,6 +184,7 @@ async fn resolve_terminal_cutover_before_commit_window( &context.lease.lease_id, &preparation, context.now_ms, + HlsTerminalTailCompatibility::TerminalMediaNotReady, ), context.evaluation.safe_deadline, ); @@ -239,7 +240,7 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover( lease: &HlsAccessLease, now_ms: u64, ) -> HlsTerminalResolution { - commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( + let detailed = commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( ctx, session, proxy_session_id, @@ -247,8 +248,30 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover( now_ms, HlsTerminalDecisionPurpose::OrdinaryCutover, ) - .await - .resolution + .await; + if detailed.live_reserve_deadline.is_some() + && matches!(detailed.resolution, HlsTerminalResolution::LiveAllowed | HlsTerminalResolution::Reevaluate) + { + let registration = + super::reevaluation::register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(session)).await; + if matches!( + registration, + crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::CapacityExceeded + | crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::RuntimeUnavailable + ) { + return commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( + ctx, + session, + proxy_session_id, + lease, + now_ms, + HlsTerminalDecisionPurpose::AutonomousOwnerFailureFallback, + ) + .await + .resolution; + } + } + detailed.resolution } pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( @@ -341,13 +364,26 @@ async fn commit_prepared_terminal_decision( preparation: &HlsTerminalTailPreparation, now_ms: u64, ) -> HlsTerminalResolution { + if !preparation.trigger.is_runtime_policy() + && now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + { + return commit_late_custom_response(HlsTerminalCommitContext { + ctx, + session, + proxy_session_id, + lease_id, + preparation, + now_ms, + }) + .await; + } let reason = preparation.trigger.reason(); let asset = match snapshot_hls_runtime_custom_tail_asset(ctx, reason) { Ok(asset) => asset, Err(compatibility) => { let context = HlsTerminalCommitContext { ctx, session, proxy_session_id, lease_id, preparation, now_ms }; let outcome = commit_terminal_unavailable(context, None, compatibility.terminal_compatibility()); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } }; commit_prepared_runtime_custom_tail(ctx, session, proxy_session_id, lease_id, preparation, now_ms, asset).await @@ -366,6 +402,11 @@ pub async fn commit_prepared_runtime_custom_tail( if preparation.trigger.reason() != asset.reason { return HlsTerminalResolution::Reevaluate; } + if !preparation.trigger.is_runtime_policy() + && now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + { + return commit_late_custom_response(context).await; + } let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&asset); let media_asset = asset.asset; let static_incompatibility = if preparation.manifest_snapshot.delivery_mode @@ -381,7 +422,7 @@ pub async fn commit_prepared_runtime_custom_tail( }; if let Some(reason) = static_incompatibility { let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } let bundle_key = prepared_terminal_bundle_key( &media_asset, @@ -403,7 +444,7 @@ pub async fn commit_prepared_runtime_custom_tail( Some(expected_asset), HlsTerminalTailCompatibility::AssetRevisionMismatch, ); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } HlsPreparedTerminalBundleState::Preparing { .. } => { return register_terminal_pending_owner(context, media_asset, expected_asset, bundle_key).await; @@ -411,12 +452,12 @@ pub async fn commit_prepared_runtime_custom_tail( HlsPreparedTerminalBundleState::Failed { key, reason } => { let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason); let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } HlsPreparedTerminalBundleState::Incompatible { reason, .. } => { let reason = terminal_bundle_incompatibility(reason); let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } }; if preparation.trigger.is_runtime_policy() { @@ -426,10 +467,11 @@ pub async fn commit_prepared_runtime_custom_tail( expected_asset, bundle_key, prepared_bundle, - ); + ) + .await; } let outcome = commit_ready_terminal_bundle(context, media_asset, expected_asset, bundle_key, prepared_bundle).await; - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } struct HlsReadyTerminalSplice { @@ -624,7 +666,7 @@ pub(super) async fn commit_ready_terminal_bundle( } } -pub(super) fn terminal_bundle_incompatibility( +pub(crate) fn terminal_bundle_incompatibility( reason: HlsPreparedTerminalBundleIncompatibility, ) -> HlsTerminalTailCompatibility { match reason { @@ -636,7 +678,7 @@ pub(super) fn terminal_bundle_incompatibility( } } -pub(super) fn terminal_bundle_failure_compatibility( +pub(crate) fn terminal_bundle_failure_compatibility( actual_key: HlsPreparedTerminalBundleKey, required_key: HlsPreparedTerminalBundleKey, reason: HlsPreparedTerminalBundleFailure, diff --git a/backend/hls/src/availability/terminal_pending_owner.rs b/backend/hls/src/availability/terminal_pending_owner.rs index 7a2f88fb1..00ad7f6fc 100644 --- a/backend/hls/src/availability/terminal_pending_owner.rs +++ b/backend/hls/src/availability/terminal_pending_owner.rs @@ -89,7 +89,7 @@ pub(super) async fn register_terminal_pending_owner( ownership, ) }); - terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration) + terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await } fn terminal_pending_owner_key( @@ -117,7 +117,7 @@ fn terminal_pending_owner_key( }) } -pub(super) fn register_ready_terminal_pending_owner( +pub(super) async fn register_ready_terminal_pending_owner( context: HlsTerminalCommitContext<'_>, asset: Arc, expected_asset: HlsRuntimeCustomTailAssetIdentity, @@ -150,16 +150,15 @@ pub(super) fn register_ready_terminal_pending_owner( }; let outcome = commit_ready_terminal_bundle(task_context, asset, expected_asset, bundle_key, bundle).await; if ownership.is_current() { - observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback( - task_context, - outcome, - )); + observe_autonomous_terminal_resolution( + terminal_resolution_with_failed_closed_fallback(task_context, outcome).await, + ); } }); - terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration) + terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await } -pub(super) fn terminal_resolution_for_pending_registration( +pub(super) async fn terminal_resolution_for_pending_registration( context: HlsTerminalCommitContext<'_>, expected_asset: HlsRuntimeCustomTailAssetIdentity, latest_safe_commit_at_ms: u64, @@ -178,7 +177,7 @@ pub(super) fn terminal_resolution_for_pending_registration( Some(expected_asset), HlsTerminalTailCompatibility::TerminalMediaNotReady, ); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } } } @@ -193,7 +192,7 @@ async fn terminal_resolution_after_settled_bundle_observation( match state { Some(HlsPreparedTerminalBundleState::Ready { bundle }) if bundle.matches_key_and_shape(bundle_key) => { let outcome = commit_ready_terminal_bundle(context, asset, expected_asset, bundle_key, bundle).await; - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Ready { .. }) => { let outcome = commit_terminal_unavailable( @@ -201,12 +200,12 @@ async fn terminal_resolution_after_settled_bundle_observation( Some(expected_asset), HlsTerminalTailCompatibility::AssetRevisionMismatch, ); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Failed { key, reason }) => { let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason); let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Incompatible { key, reason }) => { let compatibility = if key == bundle_key { @@ -215,7 +214,7 @@ async fn terminal_resolution_after_settled_bundle_observation( HlsTerminalTailCompatibility::AssetRevisionMismatch }; let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Preparing { .. }) | None => { let outcome = commit_terminal_unavailable( @@ -223,7 +222,7 @@ async fn terminal_resolution_after_settled_bundle_observation( Some(expected_asset), HlsTerminalTailCompatibility::TerminalMediaNotReady, ); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } } } @@ -330,7 +329,7 @@ pub(super) async fn run_terminal_pending_owner( commit_terminal_unavailable(context, Some(expected_asset), compatibility) } }; - observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome)); + observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome).await); } fn terminal_pending_retry_after_ms(now_ms: u64, latest_safe_commit_at_ms: u64) -> u64 { @@ -378,7 +377,7 @@ pub(super) fn terminal_resolution_for_commit_outcome( } } -pub(super) fn terminal_resolution_with_failed_closed_fallback( +pub(super) async fn terminal_resolution_with_failed_closed_fallback( context: HlsTerminalCommitContext<'_>, outcome: HlsTerminalCommitOutcome, ) -> HlsTerminalResolution { @@ -389,6 +388,9 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback( if context.preparation.trigger.is_runtime_policy() { return resolution; } + if outcome == HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed { + return commit_late_custom_response(context).await; + } commit_prepared_terminal_unavailable_after_owner_failure( context.ctx, context.session, @@ -396,9 +398,48 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback( context.lease_id, context.preparation, context.now_ms, + HlsTerminalTailCompatibility::TerminalMediaNotReady, ) } +pub(super) async fn commit_late_custom_response(context: HlsTerminalCommitContext<'_>) -> HlsTerminalResolution { + let prepared = crate::late_custom_response::prepare_late_custom_response( + context.ctx, + context.proxy_session_id, + context.lease_id, + context.preparation, + context.now_ms, + ) + .await; + let (plan, expected_asset) = match prepared { + Ok(prepared) => prepared, + Err(crate::late_custom_response::HlsLateCustomResponseError::Superseded) => { + return HlsTerminalResolution::Reevaluate + } + Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) => { + return commit_prepared_terminal_unavailable_after_owner_failure( + context.ctx, + context.session, + context.proxy_session_id, + context.lease_id, + context.preparation, + context.now_ms, + reason, + ) + } + }; + let outcome = context.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: context.session, + lease_id: context.lease_id, + proxy_session_id: context.proxy_session_id, + preparation: context.preparation, + now_ms: context.ctx.hls_proxy.terminal_commit_now_ms().max(context.now_ms), + payload: HlsTerminalCommitPayload::CustomResponse(plan), + asset_revision_guard: terminal_asset_revision_guard(context.ctx, expected_asset.reason, Some(expected_asset)), + }); + terminal_resolution_for_commit_outcome(outcome, context.now_ms) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum HlsAutonomousTerminalObservation { Committed, @@ -486,6 +527,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure( lease_id: &HlsAccessLeaseId, preparation: &HlsTerminalTailPreparation, now_ms: u64, + reason: HlsTerminalTailCompatibility, ) -> HlsTerminalResolution { let expected_asset = configured_terminal_asset_identity(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable); let outcome = ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { @@ -494,9 +536,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure( proxy_session_id, preparation, now_ms: ctx.hls_proxy.terminal_commit_now_ms().max(now_ms), - payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure( - HlsTerminalTailCompatibility::TerminalMediaNotReady, - ), + payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure(reason), asset_revision_guard: terminal_asset_revision_guard( ctx, HlsRuntimeCustomTailReason::ChannelUnavailable, diff --git a/backend/hls/src/availability/tests.rs b/backend/hls/src/availability/tests.rs index bf7f3bd80..3b81087c2 100644 --- a/backend/hls/src/availability/tests.rs +++ b/backend/hls/src/availability/tests.rs @@ -212,6 +212,7 @@ mod policy; mod publication; +mod late_custom_response; mod recovery; mod storage; diff --git a/backend/hls/src/availability/tests/late_custom_response.rs b/backend/hls/src/availability/tests/late_custom_response.rs new file mode 100644 index 000000000..fc3f001cd --- /dev/null +++ b/backend/hls/src/availability/tests/late_custom_response.rs @@ -0,0 +1,532 @@ +use super::*; +use crate::{ + late_custom_response::prepare_late_custom_response, + manager::{HlsTerminalCommitPayload, HlsTerminalCommitRequest}, +}; + +async fn late_fixture(name: &str) -> HlsTerminalPendingCommitFixture { + let fixture = terminal_pending_commit_fixture(name).await; + fixture.session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; + fixture +} + +async fn prepare( + fixture: &HlsTerminalPendingCommitFixture, +) -> (Arc, HlsRuntimeCustomTailAssetIdentity) { + prepare_late_custom_response( + &fixture.ctx, + &fixture.proxy_session_id, + &fixture.lease_id, + &fixture.preparation, + fixture.now_ms, + ) + .await + .unwrap_or_else(|_| panic!("late custom plan")) +} + +fn commit( + fixture: &HlsTerminalPendingCommitFixture, + plan: Arc, + expected_asset: HlsRuntimeCustomTailAssetIdentity, + now_ms: u64, +) -> HlsTerminalCommitOutcome { + fixture.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &fixture.session, + lease_id: &fixture.lease_id, + proxy_session_id: &fixture.proxy_session_id, + preparation: &fixture.preparation, + now_ms, + payload: HlsTerminalCommitPayload::CustomResponse(plan), + asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, expected_asset.reason, Some(expected_asset)), + }) +} + +#[tokio::test] +async fn late_custom_response_commits_without_cached_base_and_serves_frozen_media() { + let fixture = late_fixture("late-without-base").await; + let (plan, asset) = prepare(&fixture).await; + let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(asset.media); + assert!( + fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms) + .await + .is_err(), + "preparation cannot authorize uncommitted custom media" + ); + fixture.session.write().await.segments.clear(); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::Committed); + assert!(!fixture.session.read().await.has_terminal_tail_protections()); + assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:41\n")); + assert!(!plan.manifest_body().contains("/hls/shared/live/")); + let segment = fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms) + .await + .expect("committed late media"); + assert!(!segment.bytes.is_empty()); + fixture.ctx.app_config.custom_stream_response.store(None); + assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms + 1), HlsTerminalCommitOutcome::AlreadyCommitted); + let replay = fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms + 1) + .await + .expect("frozen media after config reload"); + assert_eq!(replay.bytes, segment.bytes); + assert!(fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, "wrong", 11, now_ms) + .await + .is_err()); + assert!(fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 12, now_ms) + .await + .is_err()); + assert!(fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms + 60_000) + .await + .is_err()); +} + +#[tokio::test] +async fn late_custom_response_does_not_commit_over_recovered_origin_or_changed_asset() { + for changed_asset in [false, true] { + let fixture = late_fixture(if changed_asset { "late-asset-changed" } else { "late-origin-recovered" }).await; + let (plan, asset) = prepare(&fixture).await; + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let expected = if changed_asset { + fixture.ctx.app_config.custom_stream_response.store(None); + HlsTerminalCommitOutcome::SupersededGeneration + } else { + fixture.session.write().await.origin_control.record_media_progress(now_ms, 12_000); + HlsTerminalCommitOutcome::RecoveryCommitted + }; + assert_eq!(commit(&fixture, plan, asset, now_ms), expected); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + } +} + +#[tokio::test] +async fn late_custom_response_preserves_the_prepared_asset_reason_guard() { + let fixture = late_fixture("late-wrong-asset-reason").await; + let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted) + .expect("configured alternate custom asset"); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert_eq!( + commit_prepared_runtime_custom_tail( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + &fixture.preparation, + now_ms, + asset, + ) + .await, + HlsTerminalResolution::Reevaluate + ); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("live lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); +} + +#[tokio::test] +async fn late_custom_response_preserves_target_duration_exceeded_through_terminal_commit() { + let mut fixture = late_fixture("late-target-duration-error").await; + let mut manifest = fixture.preparation.manifest_snapshot.clone(); + manifest.target_duration_ms = 1_000; + manifest.playlist_duration_ms = 1_000; + manifest.last_visible_media_end_ms = 1_000; + Arc::make_mut(&mut manifest.visible_segments)[0].duration_ms = 1_000; + let guard = fixture + .ctx + .hls_proxy + .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("publication guard"); + assert!(fixture + .ctx + .hls_proxy + .commit_access_lease_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + manifest, + fixture.now_ms, + ) + .await + .is_committed()); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("lease with short target duration"); + fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("published snapshot"); + fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation; + let expected = HlsTerminalTailCompatibility::TargetDurationExceeded { + asset_ms: fixture.asset.duration_ms(), + target_ms: 1_000, + }; + assert!(matches!(prepare_late_custom_response( + &fixture.ctx, &fixture.proxy_session_id, &fixture.lease_id, &fixture.preparation, fixture.now_ms, + ).await, Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) if reason == expected)); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert_eq!( + super::super::terminal_pending_owner::commit_late_custom_response(HlsTerminalCommitContext { + ctx: &fixture.ctx, + session: &fixture.session, + proxy_session_id: &fixture.proxy_session_id, + lease_id: &fixture.lease_id, + preparation: &fixture.preparation, + now_ms, + }) + .await, + HlsTerminalResolution::Committed + ); + let terminal = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("terminal unavailable lease"); + assert_eq!( + terminal.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { + decision_generation: fixture.preparation.decision_generation, + reason: expected, + } + ); +} + +async fn committed_late_activity_fixture(name: &str) -> HlsTerminalPendingCommitFixture { + let fixture = late_fixture(name).await; + let mut config = fixture.ctx.app_config.config.load_full().as_ref().clone(); + config.reverse_proxy = + Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto { + hls_cache: Some(shared::model::HlsCacheConfigDto { + session_idle_timeout: shared::model::Secs::new(60), + ..Default::default() + }), + ..Default::default() + })); + fixture.ctx.app_config.config.store(Arc::new(config)); + fixture.ctx.hls_proxy.update_config(&fixture.ctx.app_config).await; + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert!(fixture + .ctx + .hls_proxy + .activate_access_lease( + &fixture.lease_id, + &fixture.proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + let (plan, asset) = prepare(&fixture).await; + assert_eq!(commit(&fixture, plan, asset, now_ms), HlsTerminalCommitOutcome::Committed); + fixture +} + +#[tokio::test] +async fn late_custom_media_requests_keep_lease_and_session_alive_beyond_initial_sixty_seconds() { + use crate::{ + lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey}, + HlsMediaActivityCommitOutcome, + }; + let fixture = committed_late_activity_fixture("late-active-media").await; + let manager = &fixture.ctx.hls_proxy; + let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let before = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms) + .await + .expect("terminal lease"); + let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media); + for index in 1_u16..=11 { + let now_ms = started_at_ms + u64::from(index) * 10_000; + let access = manager + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, index, now_ms) + .await + .expect("custom media remains available"); + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, now_ms).await, + HlsMediaActivityCommitOutcome::Committed + ); + if index == 6 { + for key in [ + HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: fixture.lease_id.clone(), + proxy_session_id: fixture.proxy_session_id.clone(), + }, + HlsLifecycleEventKey::SessionIdle { proxy_session_id: fixture.proxy_session_id.clone() }, + ] { + manager + .handle_lifecycle_event( + &fixture.ctx.active_users, + &fixture.ctx.active_provider, + HlsLifecycleEvent { key, due_at_ms: started_at_ms + 60_000 }, + now_ms, + ) + .await; + } + } + let lease = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("active terminal lease"); + assert_eq!(lease.state, HlsAccessLeaseState::Activated); + assert_eq!(lease.last_seen_at_ms, now_ms); + assert_eq!(lease.valid_until_ms, now_ms + 60_000); + assert!(lease.active_until_ms.is_some_and(|until| until > now_ms)); + assert_eq!(lease.playback_mode, before.playback_mode); + assert_eq!(lease.playback_cursor, before.playback_cursor); + assert_eq!(lease.admission_generation, before.admission_generation); + assert_eq!(fixture.session.read().await.last_client_access_at_ms, now_ms); + } + let expired_at_ms = started_at_ms + 170_000; + assert!(manager + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, expired_at_ms) + .await + .is_err()); + let expired = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expired_at_ms) + .await + .expect("expired lease"); + assert_eq!(expired.state, HlsAccessLeaseState::Expired); + assert_eq!(expired.last_seen_at_ms, started_at_ms + 110_000); +} + +#[tokio::test] +async fn late_custom_media_activity_reactivates_idle_lease_but_rejects_stale_or_expired_access() { + use crate::{ + lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey}, + HlsMediaActivityCommitOutcome, + }; + let fixture = committed_late_activity_fixture("late-idle-media").await; + let manager = &fixture.ctx.hls_proxy; + let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + manager + .handle_lifecycle_event( + &fixture.ctx.active_users, + &fixture.ctx.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseActive { + lease_id: fixture.lease_id.clone(), + proxy_session_id: fixture.proxy_session_id.clone(), + }, + due_at_ms: started_at_ms + 30_000, + }, + started_at_ms + 40_000, + ) + .await; + let idle = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) + .await + .expect("idle lease"); + assert_eq!(idle.state, HlsAccessLeaseState::Idle); + let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media); + let mut access = manager + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, started_at_ms + 40_000) + .await + .expect("idle custom media"); + let original_generation = access.terminal_generation; + access.terminal_generation = Some(crate::HlsTerminalTailGeneration(fixture.preparation.decision_generation + 1)); + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + ); + let unchanged = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) + .await + .expect("unchanged idle lease"); + assert_eq!(unchanged.last_seen_at_ms, idle.last_seen_at_ms); + assert_eq!(unchanged.valid_until_ms, idle.valid_until_ms); + access.terminal_generation = original_generation; + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await, + HlsMediaActivityCommitOutcome::Committed + ); + let active = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) + .await + .expect("reactivated lease"); + assert_eq!(active.state, HlsAccessLeaseState::Activated); + assert_eq!(active.valid_until_ms, started_at_ms + 100_000); + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 100_000).await, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + ); + let expired = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 100_000) + .await + .expect("expired lease"); + assert_eq!(expired.state, HlsAccessLeaseState::Expired); + assert_eq!(expired.last_seen_at_ms, active.last_seen_at_ms); + assert_eq!(expired.valid_until_ms, active.valid_until_ms); +} + +#[tokio::test] +async fn late_custom_response_retries_contended_commit_once_and_remains_immutable() { + let mut fixture = late_fixture("late-contended-commit").await; + let now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms(); + fixture.preparation.cutover_timing = + HlsLeaseCutoverTiming::from_reserve(now_ms, 0, fixture.preparation.reserve.transition_margin, None); + let (plan, asset) = prepare(&fixture).await; + let guard = fixture.session.read().await; + assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. })); + assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. })); + drop(guard); + let committed = tokio::time::timeout(Duration::from_secs(3), async { + loop { + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + fixture.ctx.hls_proxy.terminal_commit_now_ms(), + ) + .await + .expect("lease"); + if let HlsLeasePlaybackMode::TerminalCustomResponse(committed) = lease.playback_mode { + break committed; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("autonomous late custom commit"); + assert_eq!(committed, plan); + assert_eq!( + commit(&fixture, plan, asset, fixture.ctx.hls_proxy.terminal_commit_now_ms()), + HlsTerminalCommitOutcome::AlreadyCommitted + ); +} + +#[tokio::test] +async fn late_custom_response_uses_highwater_even_when_latest_manifest_is_shorter() { + let mut fixture = late_fixture("late-highwater").await; + for (last_seq, discontinuity_sequence, discontinuity_before) in [(100, 7, true), (50, 3, false)] { + let guard = fixture + .ctx + .hls_proxy + .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("publication guard"); + let mut manifest = fixture.preparation.manifest_snapshot.clone(); + manifest.first_proxy_seq = last_seq; + manifest.last_proxy_seq = last_seq; + manifest.discontinuity_sequence = discontinuity_sequence; + let segment = Arc::make_mut(&mut manifest.visible_segments).first_mut().expect("segment"); + segment.proxy_seq = last_seq; + segment.discontinuity_before = discontinuity_before; + assert!(fixture + .ctx + .hls_proxy + .commit_access_lease_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + manifest, + fixture.now_ms + ) + .await + .is_committed()); + } + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("lease"); + fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("latest snapshot"); + fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation; + let (plan, _) = prepare(&fixture).await; + assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:101\n")); + assert!(plan.manifest_body().contains("#EXT-X-DISCONTINUITY-SEQUENCE:8\n")); +} + +#[tokio::test] +async fn late_custom_response_cas_budget_cannot_be_renewed_or_poisoned_by_polls() { + use crate::terminal_commit::{ + terminal_commit_retry_schedule_budget_ms, HlsTerminalCommitCommand, HlsTerminalCommitOwnerKey, + HlsTerminalCommitRetryCoordinator, HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision, + }; + let fixture = late_fixture("late-bounded-cas").await; + let (plan, asset) = prepare(&fixture).await; + let coordinator = HlsTerminalCommitRetryCoordinator::default(); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let command = |decision| { + let (cancellation_epoch, submission_token) = coordinator.reserve_submission().expect("submission binding"); + HlsTerminalCommitCommand { + key: HlsTerminalCommitOwnerKey { + proxy_session_id: fixture.proxy_session_id.clone(), + lease_id: fixture.lease_id.clone(), + lease_issued_at_ms: fixture.preparation.lease_issued_at_ms, + decision_generation: fixture.preparation.decision_generation, + }, + session: Arc::clone(&fixture.session), + session_incarnation: fixture + .ctx + .hls_proxy + .sessions() + .session_incarnation(&fixture.session) + .expect("session incarnation"), + preparation: fixture.preparation.clone(), + decision, + media_guard: None, + asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, asset.reason, Some(asset)), + cancellation_epoch, + submission_token, + } + }; + assert!(matches!( + coordinator.submit( + command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)), + now_ms + ), + HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + )); + assert!(matches!( + coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms), + HlsTerminalCommitSubmissionDecision::Attempt { .. } + )); + assert!(matches!( + coordinator.submit( + command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)), + now_ms + 1 + ), + HlsTerminalCommitSubmissionDecision::PendingExisting { .. } + )); + assert!(matches!( + coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms + 100), + HlsTerminalCommitSubmissionDecision::PendingExisting { .. } + )); + assert!(matches!( + coordinator.submit( + command(HlsTerminalLeaseDecision::CustomResponse(plan)), + now_ms + terminal_commit_retry_schedule_budget_ms() + ), + HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + )); +} diff --git a/backend/hls/src/availability/tests/owner_fixtures.rs b/backend/hls/src/availability/tests/owner_fixtures.rs index 3ae6a2e53..9a5860cd5 100644 --- a/backend/hls/src/availability/tests/owner_fixtures.rs +++ b/backend/hls/src/availability/tests/owner_fixtures.rs @@ -31,6 +31,7 @@ pub(super) fn post_refresh_owner_request(fixture: &PostRefreshTerminalFixture) - manifest_commit_requirement: super::super::super::refresh::HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, now_ms: fixture.now_ms, diff --git a/backend/hls/src/availability/tests/recovery.rs b/backend/hls/src/availability/tests/recovery.rs index 218a42c31..cebe93d8f 100644 --- a/backend/hls/src/availability/tests/recovery.rs +++ b/backend/hls/src/availability/tests/recovery.rs @@ -386,3 +386,287 @@ fn hls_session_recovery_pressure_new_publication_lateness_uses_current_reserve_e assert!(pressure.decision.close_admission); assert!(!pressure.decision.evaluate_lease_cutovers); } + +async fn wait_for_terminal_mode(fixture: &PostRefreshTerminalFixture) -> HlsLeasePlaybackMode { + tokio::time::timeout(Duration::from_millis(4_500), async { + loop { + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("active lease"); + if lease.playback_mode != HlsLeasePlaybackMode::Live { + return lease.playback_mode; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("terminal decision before reserve is exhausted") +} + +#[tokio::test] +async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh_is_in_flight() { + let fixture = post_refresh_terminal_fixture_with_progress("autonomous-reserve-deadline", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 18_000, + fixture.now_ms, + ) + .await; + { + let mut session = fixture.session.write().await; + session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure; + session.origin_refresh.mark_started(fixture.now_ms); + } + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + let result = commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await; + assert_eq!(result, HlsTerminalResolution::LiveAllowed); + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); + + // R=17,100 ms and M=12,000 ms: the next six-second client poll would be too late. + // Only read the resulting state; do not issue another terminal/manifest request. + let mode = wait_for_terminal_mode(&fixture).await; + assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}"); + assert!(current_time_millis() < fixture.now_ms + 5_100); + assert!(fixture.session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let fixture = post_refresh_terminal_fixture_with_progress("first-http-failure-cutover", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 18_000, + fixture.now_ms, + ) + .await; + fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms, 12_000); + let hits = Arc::new(AtomicUsize::new(0)); + let handler_hits = Arc::clone(&hits); + let router = axum::Router::new().route( + "/index.m3u8", + axum::routing::get(move || { + let handler_hits = Arc::clone(&handler_hits); + async move { + let status = if handler_hits.fetch_add(1, Ordering::SeqCst) == 0 { + axum::http::StatusCode::BAD_GATEWAY + } else { + axum::http::StatusCode::NOT_FOUND + }; + (status, [(axum::http::header::RETRY_AFTER, "60")], "origin failure") + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("local origin binds"); + let address = listener.local_addr().expect("local origin address"); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.expect("local origin serves"); + }); + let mut request = post_refresh_owner_request(&fixture); + request.origin_entry = crate::manifest_fetch::LiveHlsOriginEntry::parse(&format!("http://{address}/index.m3u8")) + .expect("local origin URL"); + let refresh = tokio::spawn(crate::refresh::trigger_origin_refresh_sync(request)); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if fixture.session.read().await.origin_control.path_condition + == HlsOriginPathCondition::RetryableFetchFailure + && fixture.ctx.hls_proxy.availability_reevaluations().owner_count() == 1 + { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("first failed attempt publishes degraded evidence and owns the deadline"); + assert_eq!(hits.load(Ordering::SeqCst), 1); + assert!(fixture.session.read().await.origin_refresh.in_flight); + + let mode = wait_for_terminal_mode(&fixture).await; + assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}"); + assert!(current_time_millis() < fixture.now_ms + 5_100); + assert_eq!(hits.load(Ordering::SeqCst), 1, "cutover must precede the capped six-second retry"); + assert!(fixture.session.read().await.origin_refresh.in_flight); + assert!(tokio::time::timeout(Duration::from_secs(8), refresh) + .await + .expect("refresh finishes") + .expect("refresh task")); + server.abort(); +} + +#[tokio::test] +async fn recovered_origin_cancels_the_retained_reserve_cutover() { + let fixture = post_refresh_terminal_fixture_with_progress("recovered-reserve-owner", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 18_000, + fixture.now_ms, + ) + .await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::LiveAllowed + ); + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); + fixture.session.write().await.origin_control.record_media_progress(current_time_millis(), 12_000); + fixture.ctx.hls_proxy.availability_reevaluations().notify_session_evidence_changed(&fixture.proxy_session_id); + wait_for_availability_owner_completion(&fixture).await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("recovered lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); +} + +#[tokio::test] +async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leaving_a_live_lease() { + let fixture = post_refresh_terminal_fixture_with_progress("expired-splice-unavailable", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 26_000, + fixture.now_ms, + ) + .await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::Committed + ); + let terminal = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("terminal lease"); + assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &terminal, + current_time_millis(), + ) + .await, + HlsTerminalResolution::Committed + ); + assert!(!fixture.session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test(start_paused = true)] +async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_losing_refresh_work() { + let fixture = post_refresh_terminal_fixture_with_progress("reserve-owner-recovery-upgrade", true, false).await; + fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::LiveAllowed + ); + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); + assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await; +} + +#[tokio::test] +async fn publication_late_origin_can_finish_custom_response_after_the_splice_deadline() { + let fixture = post_refresh_terminal_fixture_with_progress("publication-late-unavailable", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 26_000, + fixture.now_ms, + ) + .await; + fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms - 18_000, 12_000); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::Committed + ); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("terminal lease"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); +} diff --git a/backend/hls/src/availability/tests/runtime_fixtures.rs b/backend/hls/src/availability/tests/runtime_fixtures.rs index 4305fb8e9..a85538356 100644 --- a/backend/hls/src/availability/tests/runtime_fixtures.rs +++ b/backend/hls/src/availability/tests/runtime_fixtures.rs @@ -63,6 +63,7 @@ pub(super) async fn wait_for_runtime_custom_plan(fixture: &PostRefreshTerminalFi let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode { HlsLeasePlaybackMode::Live => "live", HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", + HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response", HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", HlsLeasePlaybackMode::Ended => "ended", }); @@ -332,7 +333,7 @@ pub(super) async fn assert_multi_lease_fallback_handles_pending_and_unavailable( assert_eq!(aggregate.total, 2); assert_eq!(aggregate.pending_owned, 1); assert_eq!(aggregate.terminal_committed, 1, "{aggregate:?}"); - assert!(aggregate.unresolved.is_empty()); + assert_eq!(aggregate.unresolved.len(), 0); assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1); let unavailable = fixture .ctx diff --git a/backend/hls/src/availability/tests/terminal_fixtures.rs b/backend/hls/src/availability/tests/terminal_fixtures.rs index 6d0167694..6b5fc6dca 100644 --- a/backend/hls/src/availability/tests/terminal_fixtures.rs +++ b/backend/hls/src/availability/tests/terminal_fixtures.rs @@ -148,6 +148,7 @@ pub(super) async fn install_terminal_pending_ready_base( host_local_index: 0, }, proxy_seq: 40, + media_start_ms: 0, duration_ms: asset.duration_ms(), proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -316,7 +317,8 @@ pub(super) async fn assert_terminal_pending_registration_failure_commits_unavail fixture.expected_asset, fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), failure, - ); + ) + .await; assert_eq!(resolution, HlsTerminalResolution::Committed); let lease = fixture @@ -340,6 +342,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l let mut session = fixture.session.write().await; session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; let origin_epoch = session.origin_control.origin_epoch; + let base_end_ms = session.segments[&40].media_start_ms + session.segments[&40].duration_ms; for proxy_seq in 41_u64..=44 { session.segments.insert( proxy_seq, @@ -351,6 +354,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l host_local_index: u32::try_from(proxy_seq.saturating_sub(40)).unwrap_or(u32::MAX), }, proxy_seq, + media_start_ms: base_end_ms + (proxy_seq - 41) * 20_000, duration_ms: 20_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/hls/src/availability_reevaluation.rs b/backend/hls/src/availability_reevaluation.rs index 94e4d8d2c..721c947df 100644 --- a/backend/hls/src/availability_reevaluation.rs +++ b/backend/hls/src/availability_reevaluation.rs @@ -81,6 +81,8 @@ pub enum HlsAvailabilityReevaluationRegistration { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum HlsAvailabilityReevaluationMode { RecoveryPressure, + /// Tracks a live reserve deadline independently of a client request or a running origin fetch. + TerminalReserve, PostRefresh(HlsPostRefreshAvailabilityReason), } @@ -90,8 +92,10 @@ impl HlsAvailabilityReevaluationMode { (Self::PostRefresh(current), Self::PostRefresh(incoming)) => { Self::PostRefresh(stronger_post_refresh_reason(current, incoming)) } - (Self::PostRefresh(reason), Self::RecoveryPressure) - | (Self::RecoveryPressure, Self::PostRefresh(reason)) => Self::PostRefresh(reason), + (Self::PostRefresh(reason), Self::RecoveryPressure | Self::TerminalReserve) + | (Self::RecoveryPressure | Self::TerminalReserve, Self::PostRefresh(reason)) => Self::PostRefresh(reason), + (Self::TerminalReserve, Self::RecoveryPressure | Self::TerminalReserve) + | (Self::RecoveryPressure, Self::TerminalReserve) => Self::TerminalReserve, (Self::RecoveryPressure, Self::RecoveryPressure) => Self::RecoveryPressure, } } @@ -106,10 +110,14 @@ const fn stronger_post_refresh_reason( | (_, HlsPostRefreshAvailabilityReason::HardManifestFailure) => { HlsPostRefreshAvailabilityReason::HardManifestFailure } + (HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, _) + | (_, HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict) => { + HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict + } ( - HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - ) => HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, + HlsPostRefreshAvailabilityReason::RetryableManifestFailure, + HlsPostRefreshAvailabilityReason::RetryableManifestFailure, + ) => HlsPostRefreshAvailabilityReason::RetryableManifestFailure, } } @@ -174,7 +182,7 @@ impl HlsAvailabilityReevaluationCoordinator { pub fn register( self: &Arc, key: HlsAvailabilityReevaluationOwnerKey, - mode: HlsAvailabilityReevaluationMode, + mut mode: HlsAvailabilityReevaluationMode, work: Work, ) -> HlsAvailabilityReevaluationRegistration where @@ -186,24 +194,25 @@ impl HlsAvailabilityReevaluationCoordinator { }; let mut state = self.lock_state(); if let Some(current) = state.owners.get_mut(&key.proxy_session_id) { - if current.key == key { - current.mode = current.mode.merge(mode); - current.rerun_requested = true; - current.wake.notify_one(); - return HlsAvailabilityReevaluationRegistration::AlreadyOwned; - } - if !key.supersedes(¤t.key) { + if current.key != key && !key.supersedes(¤t.key) { return HlsAvailabilityReevaluationRegistration::Superseded; } + let installs_refresh_payload = current.mode == HlsAvailabilityReevaluationMode::TerminalReserve + && mode != HlsAvailabilityReevaluationMode::TerminalReserve; + mode = current.mode.merge(mode); // A newer demand for the same proxy session is owned by the // already-counted task. The worker will snapshot current session // evidence and atomically rekey itself without acquiring another // permit. Keeping one dirty bit preserves a bounded successor // cycle while coalescing repeated registrations. - current.mode = current.mode.merge(mode); - current.rerun_requested = true; - current.wake.notify_one(); - return HlsAvailabilityReevaluationRegistration::AlreadyOwned; + // A reserve-only worker has no origin request payload. Replace it + // when recovery work arrives so merging the mode cannot discard that work. + if !installs_refresh_payload { + current.mode = mode; + current.rerun_requested = true; + current.wake.notify_one(); + return HlsAvailabilityReevaluationRegistration::AlreadyOwned; + } } let Ok(task_permit) = Arc::clone(&self.task_capacity).try_acquire_owned() else { return HlsAvailabilityReevaluationRegistration::CapacityExceeded; diff --git a/backend/hls/src/cache.rs b/backend/hls/src/cache.rs index 48099555f..b4497d47e 100644 --- a/backend/hls/src/cache.rs +++ b/backend/hls/src/cache.rs @@ -48,6 +48,8 @@ pub struct StagedCacheObject { pub size: u64, cache_path_generation: Arc, _registration: Arc, + pub(crate) write_budget: Option>, + cleanup: Arc, } /// Typed source used to distinguish a decoded-object size violation from a filesystem failure. @@ -106,6 +108,10 @@ pub struct HlsSegmentCache { capacity_reclaimer: StdRwLock>>, capacity_admission_gate: AsyncMutex<()>, owned_operation_permits: Arc, + #[cfg(test)] + startup_test_checkpoint: StdMutex>, + #[cfg(test)] + delete_test_failures: StdMutex>, } #[derive(Clone)] @@ -125,8 +131,13 @@ struct CacheCapacityState { revision: Arc, active_mutations: HashSet, revision_reservations: HashMap, - /// Revision file bytes found by the last completed usage scan. + next_write_id: u64, + write_owners: HashMap, + abandoned_writes: HashMap, + reserved_spools: HashMap, + /// Current revision bytes, reconciled by scans and successful file mutations. revision_bytes: u64, + revision_metrics: Option>, } #[derive(Default)] @@ -135,17 +146,24 @@ struct CapacityRevision; #[derive(Default)] struct TempFileState { active_files: HashSet, + revision: Arc, deletion_reservations: HashSet, } struct ActiveTempFileRegistration { path: PathBuf, state: Arc>, + changed: Arc, } impl Drop for ActiveTempFileRegistration { fn drop(&mut self) { - self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_files.remove(&self.path); + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.active_files.remove(&self.path) { + state.revision = Arc::new(CapacityRevision); + drop(state); + self.changed.notify_waiters(); + } } } @@ -171,6 +189,8 @@ struct CapacityMutationReservation { session_component: String, replacement: Option<(u64, u64)>, filesystem_mutation_started: bool, + write_budget: Option>, + transferred_bytes: u64, capacity: Arc>, changed: Arc, } @@ -198,6 +218,8 @@ pub struct HlsRevisionDiskReservation { changed: Arc, /// Set when bytes were written under this reservation outside the staged commit accounting. unaccounted_writes: bool, + generation: Arc, + cache_path: PathBuf, } pub struct HlsRevisionFilePin { @@ -217,6 +239,9 @@ mod lifecycle; mod revisions; mod staging; #[cfg(test)] +mod test_support; +mod write_budget; +#[cfg(test)] use capacity::CapacityReservationError; #[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] pub use capacity::{ @@ -226,3 +251,6 @@ pub use capacity::{ pub use error::{hls_cache_capacity_from_io, hls_cache_object_limit_from_io}; pub use keys::HlsCacheObjectKey; pub use staging::CachedSegmentMetadata; +#[cfg(test)] +pub(crate) use test_support::StartupWritePhase; +use write_budget::StagedFileCleanup; diff --git a/backend/hls/src/cache/capacity.rs b/backend/hls/src/cache/capacity.rs index dbb503968..cbd3fca08 100644 --- a/backend/hls/src/cache/capacity.rs +++ b/backend/hls/src/cache/capacity.rs @@ -1,7 +1,7 @@ use super::{ error::{cache_object_limit_error, capacity_error}, hls_cache_capacity_from_io, - lifecycle::{is_temp_cache_file, REWRITE_SECRET_FINGERPRINT_FILE}, + lifecycle::REWRITE_SECRET_FINGERPRINT_FILE, revisions::{is_revision_cache_path, revision_reserved_bytes}, safe_proxy_session_id, CacheCapacityPressure, CacheCapacityState, CapacityInvalidationGuard, CapacityMutationReservation, CapacityRevision, HlsCacheCapacityRevision, HlsCacheObjectKey, HlsSegmentCache, @@ -82,11 +82,21 @@ impl CapacityMutationReservation { return Err(CapacityReservationError::Invalidated); } let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); - let projected_total = capacity.total_bytes.saturating_sub(old_size).saturating_add(new_size); - let projected_session = session_size.saturating_sub(old_size).saturating_add(new_size); + // Old bytes remain on disk until rename succeeds. Reserve growth now, + // but publish any shrink only after the filesystem mutation completes. + let growth = new_size.saturating_sub(old_size); + let projected_total = capacity.total_bytes.saturating_add(growth); + let projected_session = session_size.saturating_add(growth); let (reserved_global, reserved_session) = revision_reserved_bytes(&capacity, &self.session_component); - if projected_total.saturating_add(reserved_global) > max_cache_bytes - || projected_session.saturating_add(reserved_session) > max_session_bytes + let transferred = self + .write_budget + .as_ref() + .and_then(|budget| { + capacity.revision_reservations.get(&budget.id).filter(|(owner, _)| owner == &self.session_component) + }) + .map_or(0, |(_, bytes)| (*bytes).min(new_size.saturating_sub(old_size))); + if projected_total.saturating_add(reserved_global.saturating_sub(transferred)) > max_cache_bytes + || projected_session.saturating_add(reserved_session.saturating_sub(transferred)) > max_session_bytes { return Err(CapacityReservationError::Exceeded { pressure: CacheCapacityPressure { @@ -96,10 +106,10 @@ impl CapacityMutationReservation { current_global_bytes: capacity.total_bytes, staged_bytes: new_size, required_session_bytes: projected_session - .saturating_add(reserved_session) + .saturating_add(reserved_session.saturating_sub(transferred)) .saturating_sub(max_session_bytes), required_global_bytes: projected_total - .saturating_add(reserved_global) + .saturating_add(reserved_global.saturating_sub(transferred)) .saturating_sub(max_cache_bytes), }, // Capture the opaque token in the same critical section as the @@ -108,6 +118,12 @@ impl CapacityMutationReservation { revision: HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), }); } + if let Some(budget) = &self.write_budget { + if let Some((_, bytes)) = capacity.revision_reservations.get_mut(&budget.id) { + *bytes -= transferred; + } + } + self.transferred_bytes = transferred; capacity.total_bytes = projected_total; store_session_bytes(&mut capacity, &self.session_component, projected_session); self.replacement = Some((old_size, new_size)); @@ -117,12 +133,36 @@ impl CapacityMutationReservation { pub(super) fn mark_filesystem_mutation_started(&mut self) { self.filesystem_mutation_started = true; } pub(super) fn finish_replacement(mut self) { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some((old, new)) = self.replacement { + let released = old.saturating_sub(new); + capacity.total_bytes = capacity.total_bytes.saturating_sub(released); + let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); + store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(released)); + if super::revisions::is_revision_cache_path(&self.path) { + capacity.revision_bytes = capacity.revision_bytes.saturating_sub(old).saturating_add(new); + } + } + capacity.abandoned_writes.remove(&self.path); + if let Some(budget) = &self.write_budget { + capacity.write_owners.insert(self.path.clone(), budget.id); + } else { + capacity.write_owners.remove(&self.path); + } + drop(capacity); self.replacement = None; self.finish_mutation(None); } pub(super) fn finish_delete(mut self, deleted_size: u64) { - self.finish_mutation(Some(deleted_size)); + let unaccounted = self + .capacity + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reserved_spools + .remove(&self.path) + .is_some(); + self.finish_mutation(Some(if unaccounted { 0 } else { deleted_size })); self.replacement = None; } @@ -134,11 +174,17 @@ impl CapacityMutationReservation { capacity.active_mutations.remove(&self.path); if capacity.initialized && capacity.cache_path == self.cache_path { if let Some(deleted_size) = deleted_size { + capacity.write_owners.remove(&self.path); + capacity.abandoned_writes.remove(&self.path); + if super::revisions::is_revision_cache_path(&self.path) { + capacity.revision_bytes = capacity.revision_bytes.saturating_sub(deleted_size); + } capacity.total_bytes = capacity.total_bytes.saturating_sub(deleted_size); let session_bytes = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); store_session_bytes(&mut capacity, &self.session_component, session_bytes.saturating_sub(deleted_size)); } } + update_revision_metrics(&capacity); capacity.revision = Arc::new(CapacityRevision); drop(capacity); self.changed.notify_waiters(); @@ -152,18 +198,22 @@ impl Drop for CapacityMutationReservation { return; } capacity.active_mutations.remove(&self.path); + if !self.filesystem_mutation_started && self.transferred_bytes > 0 { + if let Some((_, bytes)) = + self.write_budget.as_ref().and_then(|budget| capacity.revision_reservations.get_mut(&budget.id)) + { + *bytes = bytes.saturating_add(self.transferred_bytes); + } + } let revision_changed = if self.filesystem_mutation_started { capacity.initialized = false; true } else if capacity.initialized && capacity.cache_path == self.cache_path { if let Some((old_size, new_size)) = self.replacement.take() { - capacity.total_bytes = capacity.total_bytes.saturating_sub(new_size).saturating_add(old_size); + let growth = new_size.saturating_sub(old_size); + capacity.total_bytes = capacity.total_bytes.saturating_sub(growth); let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); - store_session_bytes( - &mut capacity, - &self.session_component, - session_size.saturating_sub(new_size).saturating_add(old_size), - ); + store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(growth)); true } else { false @@ -342,13 +392,36 @@ impl HlsSegmentCache { notified.await; continue; }; - let (total_bytes, session_bytes, revision_bytes) = scan_committed_cache_usage(cache_path).await?; + let (temp_revision, mut excluded) = { + let temps = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let excluded: std::collections::HashSet<_> = temps + .active_files + .iter() + .filter(|path| super::lifecycle::is_temp_cache_file(path)) + .cloned() + .collect(); + (Arc::clone(&temps.revision), excluded) + }; + excluded.extend( + self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reserved_spools.keys().cloned(), + ); + #[cfg(test)] + self.startup_write_checkpoint(super::StartupWritePhase::AfterInventorySnapshot).await?; + let (total_bytes, session_bytes, revision_bytes) = + scan_cache_usage_excluding(cache_path, &excluded).await?; if self.cache_path_snapshot().path != cache_path { return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during usage scan")); } + // Registration precedes file creation. Changes after this check cannot + // have affected the completed scan; no nested temp/capacity locks are needed. + let temps_changed = !Arc::ptr_eq( + &self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).revision, + &temp_revision, + ); let scan_invalidated = { let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !Arc::ptr_eq(&capacity.revision, &revision) + if temps_changed + || !Arc::ptr_eq(&capacity.revision, &revision) || capacity.active_mutations.iter().any(|path| path.starts_with(cache_path)) { true @@ -358,6 +431,18 @@ impl HlsSegmentCache { capacity.total_bytes = total_bytes; capacity.session_bytes = session_bytes; capacity.revision_bytes = revision_bytes; + let sessions: std::collections::HashSet<_> = capacity.session_bytes.keys().cloned().collect(); + let active_owners: std::collections::HashSet<_> = + capacity.revision_reservations.keys().copied().collect(); + capacity.write_owners.retain(|path, owner| { + active_owners.contains(owner) + || path.strip_prefix(cache_path).ok().and_then(|path| path.components().next()).is_some_and( + |session| sessions.contains(session.as_os_str().to_string_lossy().as_ref()), + ) + }); + let owners = capacity.write_owners.clone(); + capacity.abandoned_writes.retain(|path, (owner, _)| owners.get(path) == Some(owner)); + update_revision_metrics(&capacity); false } }; @@ -407,6 +492,8 @@ impl HlsSegmentCache { session_component, replacement: None, filesystem_mutation_started: false, + write_budget: None, + transferred_bytes: 0, capacity: Arc::clone(&self.capacity), changed: Arc::clone(&self.capacity_changed), }) @@ -421,9 +508,39 @@ impl HlsSegmentCache { max_cache_bytes: u64, max_session_bytes: u64, ) -> io::Result { + self.prepare_capacity_replacement_with_budget( + cache_path, + final_path, + session_component, + new_size, + max_cache_bytes, + max_session_bytes, + None, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(super) async fn prepare_capacity_replacement_with_budget( + &self, + cache_path: &Path, + final_path: &Path, + session_component: String, + new_size: u64, + max_cache_bytes: u64, + max_session_bytes: u64, + budget: Option>, + ) -> io::Result { + if budget.as_ref().is_some_and(|budget| { + !Arc::ptr_eq(&budget.capacity, &self.capacity) + || !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation) + }) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation")); + } loop { self.ensure_capacity_initialized(cache_path).await?; let mut reservation = self.begin_capacity_mutation(cache_path, final_path, session_component.clone()).await; + reservation.write_budget = budget.clone(); let old_size = match fs::metadata(final_path).await { Ok(metadata) => metadata.len(), Err(err) if err.kind() == io::ErrorKind::NotFound => 0, @@ -465,7 +582,15 @@ fn invalidate_capacity_state(capacity: &StdMutex, changed: & changed.notify_waiters(); } +#[cfg(test)] pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<(u64, HashMap, u64)> { + scan_cache_usage_excluding(cache_path, &std::collections::HashSet::new()).await +} + +async fn scan_cache_usage_excluding( + cache_path: &Path, + excluded: &std::collections::HashSet, +) -> io::Result<(u64, HashMap, u64)> { let mut total_bytes = 0_u64; let mut revision_bytes = 0_u64; let mut session_bytes = HashMap::new(); @@ -475,22 +600,37 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result< Err(err) => return Err(err), }; while let Some(root_entry) = root_entries.next_entry().await? { - if !root_entry.file_type().await?.is_dir() { - continue; + match root_entry.file_type().await { + Ok(kind) if kind.is_dir() => {} + Ok(_) => continue, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), } let session_component = root_entry.file_name().to_string_lossy().into_owned(); let mut bytes = 0_u64; let mut pending_dirs = vec![root_entry.path()]; while let Some(dir) = pending_dirs.pop() { - let mut entries = fs::read_dir(dir).await?; + let mut entries = match fs::read_dir(dir).await { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; while let Some(entry) = entries.next_entry().await? { - let file_type = entry.file_type().await?; + let file_type = match entry.file_type().await { + Ok(kind) => kind, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; if file_type.is_dir() { pending_dirs.push(entry.path()); } else if file_type.is_file() { let revision = is_revision_cache_path(&entry.path()); - if revision || !is_temp_cache_file(&entry.path()) { - let size = entry.metadata().await?.len(); + if !excluded.contains(&entry.path()) { + let size = match entry.metadata().await { + Ok(metadata) => metadata.len(), + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; bytes = bytes.saturating_add(size); if revision { revision_bytes = revision_bytes.saturating_add(size); @@ -506,3 +646,9 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result< } Ok((total_bytes, session_bytes, revision_bytes)) } + +pub(super) fn update_revision_metrics(capacity: &CacheCapacityState) { + if let Some(metrics) = capacity.revision_metrics.as_ref().and_then(Weak::upgrade) { + metrics.record_revision_disk_bytes(capacity.revision_bytes); + } +} diff --git a/backend/hls/src/cache/lifecycle.rs b/backend/hls/src/cache/lifecycle.rs index ca6181cc7..d27540ac2 100644 --- a/backend/hls/src/cache/lifecycle.rs +++ b/backend/hls/src/cache/lifecycle.rs @@ -1,9 +1,9 @@ use super::{ - revisions::is_revision_cache_path, staging::run_owned_cache_operation, ActiveTempFileRegistration, - CacheCapacityState, CacheInvalidationOutcome, CachePathDeletionReservation, CachePathGeneration, CachePathState, - CachedSegmentMetadata, CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer, - HlsCacheCapacityRevision, HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState, - DEFAULT_HLS_CACHE_PATH, MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, + staging::run_owned_cache_operation, ActiveTempFileRegistration, CacheCapacityState, CacheInvalidationOutcome, + CachePathDeletionReservation, CachePathGeneration, CachePathState, CachedSegmentMetadata, + CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer, HlsCacheCapacityRevision, + HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState, DEFAULT_HLS_CACHE_PATH, + MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, }; use log::warn; use std::{ @@ -41,6 +41,10 @@ impl HlsSegmentCache { capacity_reclaimer: StdRwLock::new(None), capacity_admission_gate: AsyncMutex::new(()), owned_operation_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS)), + #[cfg(test)] + startup_test_checkpoint: StdMutex::new(None), + #[cfg(test)] + delete_test_failures: StdMutex::new(std::collections::HashMap::new()), } } @@ -123,6 +127,7 @@ impl HlsSegmentCache { pub async fn delete(&self, key: &K) -> io::Result<()> { let cache_path = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&cache_path.path).await?; let path = Self::path_for_key_in(&cache_path.path, key); let reservation = self.begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()).await; self.delete_with_reservation(path, reservation).await @@ -130,6 +135,7 @@ impl HlsSegmentCache { pub async fn delete_if_inactive(&self, key: &K) -> io::Result<()> { let cache_path = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&cache_path.path).await?; let path = Self::path_for_key_in(&cache_path.path, key); let reservation = self .try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()) @@ -142,6 +148,8 @@ impl HlsSegmentCache { path: PathBuf, reservation: CapacityMutationReservation, ) -> io::Result<()> { + #[cfg(test)] + self.check_cache_delete_test_failure(&path)?; run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "delete", async move { let size = match fs::metadata(&path).await { Ok(metadata) => metadata.len(), @@ -190,9 +198,7 @@ impl HlsSegmentCache { let cache_path = self.cache_path_snapshot(); let candidates = old_temp_files(&cache_path.path, cutoff, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN).await?; let mut deleted = 0_usize; - let mut removed_revision = false; for path in candidates { - let revision = is_revision_cache_path(&path); let Some(deletion) = self.reserve_path_deletion(&path) else { continue; }; @@ -209,14 +215,14 @@ impl HlsSegmentCache { match result { Ok(()) => { deleted = deleted.saturating_add(1); - removed_revision |= revision; } Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => warn!("HLS temporary cache file cleanup deferred: error_kind={:?}", error.kind()), } } - if removed_revision { + if deleted > 0 { drop(self.capacity_invalidation_guard()); + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?; } Ok(deleted) } @@ -238,7 +244,8 @@ impl HlsSegmentCache { drop(deletion); result }) - .await + .await?; + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await } pub async fn delete_orphan_session_dirs( @@ -307,6 +314,9 @@ impl HlsSegmentCache { } } } + if removed > 0 { + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?; + } Ok(removed) } @@ -389,7 +399,8 @@ impl HlsSegmentCache { drop(deletion); result }) - .await + .await?; + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await } pub async fn read_rewrite_secret_fingerprint(&self) -> io::Result> { @@ -422,7 +433,14 @@ impl HlsSegmentCache { if !state.active_files.insert(path.to_path_buf()) { return Err(io::Error::new(io::ErrorKind::AlreadyExists, "hls cache temp path is already active")); } - Ok(Arc::new(ActiveTempFileRegistration { path: path.to_path_buf(), state: Arc::clone(&self.temp_files) })) + state.revision = Arc::new(CapacityRevision); + drop(state); + self.capacity_changed.notify_waiters(); + Ok(Arc::new(ActiveTempFileRegistration { + path: path.to_path_buf(), + state: Arc::clone(&self.temp_files), + changed: Arc::clone(&self.capacity_changed), + })) } pub(super) fn reserve_path_deletion(&self, path: &Path) -> Option { diff --git a/backend/hls/src/cache/revision_capacity_tests/publication.rs b/backend/hls/src/cache/revision_capacity_tests/publication.rs index f41d09c9c..176a08b47 100644 --- a/backend/hls/src/cache/revision_capacity_tests/publication.rs +++ b/backend/hls/src/cache/revision_capacity_tests/publication.rs @@ -1,5 +1,81 @@ use super::*; +#[test] +fn temp_registration_and_last_owner_release_advance_inventory_revision() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let initial = Arc::clone(&cache.temp_files.lock().unwrap().revision); + let registration = cache.register_active_temp_path(&directory.path().join("segment.ts.tmp.test"))?; + let registered = Arc::clone(&cache.temp_files.lock().unwrap().revision); + assert!(!Arc::ptr_eq(&initial, ®istered)); + let last_owner = Arc::clone(®istration); + drop(registration); + assert!(Arc::ptr_eq(®istered, &cache.temp_files.lock().unwrap().revision)); + drop(last_owner); + assert!(!Arc::ptr_eq(®istered, &cache.temp_files.lock().unwrap().revision)); + assert!(cache.temp_files.lock().unwrap().active_files.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn attaching_a_budget_to_shared_staging_returns_an_error_and_preserves_its_owner() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("shared-staging".into()), 0, "ts"); + let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"test".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(Arc::clone(&budget)), + ) + .await?; + let error = staged.clone().with_write_budget(None).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(tokio::fs::read(&staged.path).await?, b"test"); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20); + cache.commit_staged(&key, staged).await?; + drop(budget); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 4); + Ok(()) +} + +#[tokio::test] +async fn inventory_scan_retries_when_a_reserved_temp_file_is_registered_after_its_snapshot() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = Arc::new(HlsSegmentCache::with_cache_path(directory.path())); + cache.update_cache_limits(60, 60); + let key = SegmentCacheKey::new(ProxySessionId("inventory-race".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, &[1; 20]).await?; + let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?); + cache.invalidate_capacity_accounting(); + let (reached, resume) = cache.pause_startup_write(StartupWritePhase::AfterInventorySnapshot); + let scan_cache = Arc::clone(&cache); + let id = key.proxy_session_id().clone(); + let scan = tokio::spawn(async move { scan_cache.capacity_usage(&id).await }); + tokio::time::timeout(std::time::Duration::from_secs(2), reached).await.unwrap().unwrap(); + let staged = cache + .stage_temp_with_budget( + &key, + &[2; 20][..], + Instant::now() + std::time::Duration::from_secs(2), + Some(Arc::clone(&budget)), + ) + .await?; + resume.send(()).unwrap(); + let usage = tokio::time::timeout(std::time::Duration::from_secs(2), scan).await.unwrap().unwrap()?; + cache.remove_staged(staged).await?; + drop(budget); + assert_eq!(usage.global_bytes, 20, "reserved staging bytes must not enter the committed inventory"); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 20); + let next = cache.reserve_revision_peak(&key, 30).await?; + drop(next); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + Ok(()) +} + #[tokio::test] async fn revision_peak_and_legacy_cache_share_the_global_and_session_quota() -> io::Result<()> { let directory = tempfile::tempdir()?; @@ -54,3 +130,375 @@ async fn processed_revision_retries_and_ranges_preserve_bytes_across_cache_repla assert_eq!(cache.delete_orphan_revision_files(SystemTime::now()).await?, 0); Ok(()) } + +// These are the two real disk-cache transitions from debug/bug.log. Sparse +// fixtures preserve the exact byte counts without allocating a GiB of RAM/disk. +#[tokio::test] +async fn logged_normal_commit_consumes_its_own_peak_reservation() -> io::Result<()> { + logged_peak_case(9_994_832).await +} + +#[tokio::test] +async fn logged_revision_publication_consumes_its_remaining_peak_reservation() -> io::Result<()> { + logged_peak_case(8_052_416).await +} + +async fn logged_peak_case(size: u64) -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(1_073_741_824, 1_073_741_824); + let key = SegmentCacheKey::new(ProxySessionId("logged-session".into()), 84, "ts"); + let existing = SegmentCacheKey::new(key.proxy_session_id().clone(), 0, "ts"); + let path = cache.object_path(&existing); + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + let file = tokio::fs::File::create(path).await?; + file.set_len(1_040_683_024).await?; + drop(file); + let reservation = std::sync::Arc::new(cache.reserve_revision_peak(&key, 3 * size).await?); + let staged_path = cache.object_path(&key).with_extension("ts.tmp.regression"); + let file = tokio::fs::File::create(&staged_path).await?; + file.set_len(size).await?; + drop(file); + let staged = cache.adopt_staged_file(staged_path, size)?.with_write_budget(Some(reservation.clone()))?; + let committed = cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 84, crate::SegmentRevisionKind::Processed)?; + let published = cache + .publish_processed_revision_with_budget( + &revision.revision().key, + &committed.path, + Instant::now() + std::time::Duration::from_secs(5), + Some(reservation.clone()), + ) + .await?; + assert_eq!(published.size, size); + drop(reservation); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 1_040_683_024 + 2 * size); + assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?); + Ok(()) +} + +#[tokio::test] +async fn processed_reservation_release_wakes_before_and_after_wait_registration() -> io::Result<()> { + for registered in [false, true] { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("release".into()), 0, "ts"); + let reservation = cache.reserve_revision_peak(&key, 30).await?; + assert!(!reservation.unaccounted_writes); + let revision = cache.capacity_revision(); + let waiting = cache.wait_for_capacity_change(&revision); + tokio::pin!(waiting); + if registered { + assert!(futures::poll!(waiting.as_mut()).is_pending()); + } + drop(reservation); + tokio::time::timeout(std::time::Duration::from_secs(1), waiting) + .await + .expect("releasing only a processed reservation must wake the waiter"); + } + Ok(()) +} + +#[tokio::test] +async fn concurrent_peak_owners_cannot_consume_each_others_session_or_global_budget() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = std::sync::Arc::new(HlsSegmentCache::with_cache_path(directory.path())); + cache.update_cache_limits(180, 120); + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(4)); + let mut tasks = Vec::new(); + for (session, seq) in [("one", 0), ("one", 1), ("two", 0)] { + let cache = cache.clone(); + let barrier = barrier.clone(); + tasks.push(tokio::spawn(async move { + let key = SegmentCacheKey::new(ProxySessionId(session.into()), seq, "ts"); + let peak = std::sync::Arc::new(cache.reserve_revision_peak(&key, 60).await?); + barrier.wait().await; + let staged = cache + .stage_temp_with_budget( + &key, + &[7; 20][..], + Instant::now() + std::time::Duration::from_secs(2), + Some(peak.clone()), + ) + .await?; + let metadata = cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let owner = store.create(key.proxy_session_id().clone(), seq, crate::SegmentRevisionKind::Processed)?; + cache + .publish_processed_revision_with_budget( + &owner.revision().key, + &metadata.path, + Instant::now() + std::time::Duration::from_secs(2), + Some(peak), + ) + .await?; + Ok::<_, io::Error>(()) + })); + } + barrier.wait().await; + for task in tasks { + task.await??; + } + let usage = cache.capacity_usage(&ProxySessionId("one".into())).await?; + assert_eq!(usage, HlsCacheCapacityUsage { session_bytes: 80, global_bytes: 120 }); + assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("one".into())), 0); + assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("two".into())), 0); + Ok(()) +} + +#[tokio::test] +async fn old_write_cleanup_cannot_delete_a_newer_fetch_attempt() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("replace".into()), 0, "ts"); + let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"old".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(old.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + let new = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"new".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(new.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let error = cache + .publish_processed_revision_with_budget( + &revision.revision().key, + &cache.object_path(&key), + Instant::now() + std::time::Duration::from_secs(2), + Some(old.clone()), + ) + .await + .expect_err("an old startup attempt cannot publish a replacement owned by a newer attempt"); + assert_eq!(error.kind(), io::ErrorKind::Interrupted); + assert!(cache.metadata(&revision.revision().key).await?.is_none()); + cache.delete_write_owned(&key, &old).await?; + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new"); + cache.delete_write_owned(&key, &new).await?; + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0); + Ok(()) +} + +#[tokio::test] +async fn pinned_revisions_remain_accounted_after_inventory_reconciliation_and_deletion() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let metrics = std::sync::Arc::new(crate::HlsCacheMetrics::default()); + cache.install_revision_metrics(&metrics); + let key = SegmentCacheKey::new(ProxySessionId("pinned-scan".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, &[1; 20]).await?; + let store = crate::SegmentRevisionStore::default(); + let owner = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let pin = cache.pin_revision_file(&cache.object_path(&owner.revision().key))?; + *owner.revision().file_pin.lock().map_err(|_| io::Error::other("file pin"))? = Some(pin); + cache + .publish_processed_revision( + &owner.revision().key, + &cache.object_path(&key), + Instant::now() + std::time::Duration::from_secs(2), + ) + .await?; + drop(cache.capacity_invalidation_guard()); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 40); + assert_eq!(metrics.snapshot().revision_disk_bytes, cache.scan_revision_disk_bytes().await?); + drop(owner); + cache.delete_session_dir(key.proxy_session_id()).await?; + assert_eq!(metrics.snapshot().revision_disk_bytes, 0); + assert_eq!(cache.scan_revision_disk_bytes().await?, 0); + Ok(()) +} + +#[tokio::test] +async fn failed_partial_commit_cleanup_retries_without_deleting_a_replacement() -> io::Result<()> { + for replace in [false, true] { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("cleanup-retry".into()), 0, "ts"); + let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"old".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(old.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + cache.fail_next_cache_deletes(cache.object_path(&key), 1); + assert!(cache.delete_write_owned(&key, &old).await.is_err()); + drop(old); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 3); + if replace { + cache.write_bytes_and_commit(&key, b"new").await?; + } + cache.retry_abandoned_write_deletions().await; + if replace { + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new"); + } else { + assert!(cache.metadata(&key).await?.is_none()); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0); + } + cache.retry_abandoned_write_deletions().await; + } + Ok(()) +} + +#[tokio::test] +async fn cancelled_staging_keeps_its_peak_until_pending_file_io_is_joined() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(20, 20); + let key = SegmentCacheKey::new(ProxySessionId("pending-io".into()), 0, "ts"); + let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?); + let path = cache.object_path(&key).with_extension("ts.tmp.pending"); + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + let staged = cache.adopt_staged_file(path.clone(), 0)?.with_write_budget(Some(budget.clone()))?; + let writer = staged.track_writer(tokio::fs::File::create(&path).await?); + let mut in_flight_io = writer.lock().await; + in_flight_io.as_mut().unwrap().write_all(b"pending").await?; + drop(budget); + let revision = cache.capacity_revision(); + drop(staged); // Models cancellation while the producer still owns its I/O handle. + let released = cache.wait_for_capacity_change(&revision); + tokio::pin!(released); + assert!(futures::poll!(released.as_mut()).is_pending()); + assert!(cache.reserve_revision_peak(&key, 1).await.is_err()); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20); + drop(in_flight_io); + tokio::time::timeout(std::time::Duration::from_secs(2), released).await.unwrap(); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert!(!tokio::fs::try_exists(path).await?); + assert!(!cache.has_active_temp_files()); + Ok(()) +} + +#[tokio::test] +async fn cancelled_revision_link_keeps_its_peak_until_the_owned_link_and_unlink_finish() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(20, 20); + let key = SegmentCacheKey::new(ProxySessionId("pending-link".into()), 0, "ts"); + let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"0123456789".as_slice(), + Instant::now() + std::time::Duration::from_secs(5), + Some(budget.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink); + let source = cache.object_path(&key); + let publish = cache.publish_processed_revision_with_budget( + &revision.revision().key, + &source, + Instant::now() + std::time::Duration::from_secs(5), + Some(budget), + ); + let mut publish = Box::pin(publish); + tokio::select! { + entered = entered => entered.unwrap(), + result = &mut publish => panic!("publication bypassed its link checkpoint: {result:?}"), + } + drop(publish); + assert!(cache.reserve_revision_peak(&key, 1).await.is_err()); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10); + let token = cache.capacity_revision(); + resume.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), cache.wait_for_capacity_change(&token)).await.unwrap(); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 10); + assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?); + assert!(!cache.has_active_temp_files()); + assert!(cache.metadata(&revision.revision().key).await?.is_none()); + assert_eq!(tokio::fs::read(source).await?, b"0123456789"); + Ok(()) +} + +#[tokio::test] +async fn concurrent_cache_replacement_cannot_change_a_processed_publication_source_or_its_accounted_size( +) -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(64, 64); + let key = SegmentCacheKey::new(ProxySessionId("publication-source".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, b"old").await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let source = cache.object_path(&key); + let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink); + let publication = cache.publish_processed_revision( + &revision.revision().key, + &source, + Instant::now() + std::time::Duration::from_secs(5), + ); + tokio::pin!(publication); + tokio::select! { + reached = entered => reached.unwrap(), + result = &mut publication => panic!("publication bypassed the source checkpoint: {result:?}"), + } + let replacement = cache.write_bytes_and_commit(&key, b"larger-new-cache-body"); + tokio::pin!(replacement); + assert!(futures::poll!(replacement.as_mut()).is_pending()); + resume.send(()).unwrap(); + let (published, replaced) = + tokio::time::timeout(std::time::Duration::from_secs(5), async { tokio::join!(publication, replacement) }) + .await + .expect("source lease releases before publication commit reacquires admission"); + let published = published?; + let replaced = replaced?; + assert_eq!(published.size, 3); + assert_eq!(tokio::fs::read(published.path).await?, b"old"); + assert_eq!(tokio::fs::read(replaced.path).await?, b"larger-new-cache-body"); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, replaced.size + 3); + assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + Ok(()) +} + +#[tokio::test] +async fn shrinking_replacement_keeps_old_bytes_charged_until_rename_finishes() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(110, 110); + let key = SegmentCacheKey::new(ProxySessionId("shrinking".into()), 0, "ts"); + let other = SegmentCacheKey::new(ProxySessionId("other-session".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, &[1; 100]).await?; + let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeCacheRename); + let replacement = cache.write_bytes_and_commit(&key, &[2; 10]); + tokio::pin!(replacement); + tokio::select! { + reached = entered => reached.unwrap(), + result = &mut replacement => panic!("replacement bypassed the rename checkpoint: {result:?}"), + } + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [1; 100]); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 100); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10); + assert!(cache.reserve_revision_peak(&key, 1).await.is_err()); + assert!(cache.reserve_revision_peak(&other, 1).await.is_err()); + resume.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), replacement).await.unwrap()?; + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [2; 10]); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 10); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + let released = cache.reserve_revision_peak(&other, 100).await?; + drop(released); + Ok(()) +} diff --git a/backend/hls/src/cache/revisions.rs b/backend/hls/src/cache/revisions.rs index c2661068a..ae0a8a202 100644 --- a/backend/hls/src/cache/revisions.rs +++ b/backend/hls/src/cache/revisions.rs @@ -27,9 +27,12 @@ impl HlsRevisionDiskReservation { impl Drop for HlsRevisionDiskReservation { fn drop(&mut self) { let mut capacity = self.capacity.lock_unpoisoned(); - capacity.revision_reservations.remove(&self.id); + let released = capacity.revision_reservations.remove(&self.id).is_some(); + capacity.reserved_spools.retain(|_, owner| *owner != self.id); if self.unaccounted_writes { capacity.initialized = false; + } + if released || self.unaccounted_writes { capacity.revision = Arc::new(CapacityRevision); } drop(capacity); @@ -48,6 +51,14 @@ impl HlsSegmentCache { bytes: u64, ) -> io::Result { let _admission = self.capacity_admission_gate.lock().await; + self.reserve_revision_peak_inner(key, bytes).await + } + + pub(super) async fn reserve_revision_peak_inner( + &self, + key: &K, + bytes: u64, + ) -> io::Result { let root = self.cache_path_snapshot(); self.ensure_capacity_initialized(&root.path).await?; let session = key.session_path_component(); @@ -77,19 +88,20 @@ impl HlsSegmentCache { HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), )); } - let id = loop { - let id = fastrand::u64(..); - if !capacity.revision_reservations.contains_key(&id) { - break id; - } - }; + let id = + capacity.next_write_id.checked_add(1).ok_or_else(|| io::Error::other("cache write identity exhausted"))?; + capacity.next_write_id = id; capacity.revision_reservations.insert(id, (session, bytes)); capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.capacity_changed.notify_waiters(); Ok(HlsRevisionDiskReservation { id, capacity: Arc::clone(&self.capacity), changed: Arc::clone(&self.capacity_changed), unaccounted_writes: false, + generation: root.generation, + cache_path: root.path, }) } @@ -101,18 +113,44 @@ impl HlsSegmentCache { } pub async fn create_revision_spool(&self, key: &K) -> io::Result<(File, HlsRevisionFilePin)> { + self.create_revision_spool_with_budget(key, None).await + } + + pub(crate) async fn create_revision_spool_with_budget( + &self, + key: &K, + budget: Option<&HlsRevisionDiskReservation>, + ) -> io::Result<(File, HlsRevisionFilePin)> { + let commit_gate = Arc::clone(&self.cache_path_commit_gate).read_owned().await; let root = self.cache_path_snapshot(); + if budget.is_some_and(|budget| !Arc::ptr_eq(&budget.generation, &root.generation)) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "revision reservation root retired")); + } self.ensure_cache_root_marker_for(&root.path).await?; let path = Self::path_for_key_in(&root.path, key); let pin = self.pin_revision_file(&path)?; - let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?; - fs::create_dir_all(parent).await?; - let file = OpenOptions::new().write(true).create_new(true).open(&path).await?; - Ok((file, pin)) + let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?.to_path_buf(); + let mut mutation = self.begin_capacity_mutation(&root.path, &path, key.session_path_component()).await; + if let Some(budget) = budget { + self.register_reserved_spool(path.clone(), budget); + } + super::staging::run_owned_cache_operation( + Arc::clone(&self.owned_operation_permits), + "revision-create", + async move { + fs::create_dir_all(parent).await?; + let file = OpenOptions::new().write(true).create_new(true).open(&path).await?; + mutation.finish_mutation(None); + drop(commit_gate); + Ok((file, pin)) + }, + ) + .await } pub async fn delete_orphan_revision_files(&self, cutoff: SystemTime) -> io::Result { let root = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&root.path).await?; let Some(mut sessions) = skip_not_found(fs::read_dir(&root.path).await)? else { return Ok(0) }; let mut deleted = 0usize; while let Some(session) = sessions.next_entry().await? { @@ -154,23 +192,90 @@ impl HlsSegmentCache { key: &K, source: &Path, deadline: Instant, + ) -> io::Result { + self.publish_processed_revision_with_budget(key, source, deadline, None).await + } + + #[allow(clippy::too_many_lines, reason = "Keeps source ownership, peak admission and immutable linking together.")] + pub(crate) async fn publish_processed_revision_with_budget( + &self, + key: &K, + source: &Path, + deadline: Instant, + budget: Option>, ) -> io::Result { let root = self.cache_path_snapshot(); let staged_path = Self::temp_path_for_key(key, &root.path); + let size = fs::metadata(source).await?.len(); + let checks_source_owner = budget.is_some(); + let budget = match budget { + Some(budget) => budget, + None => Arc::new(self.reserve_revision_peak_reclaiming(key, size).await?), + }; + let (size, source_lease) = loop { + if Instant::now() >= deadline { + return Err(io::Error::new(io::ErrorKind::TimedOut, "revision publication timed out")); + } + self.ensure_write_budget_available(key, &budget, size).await?; + // Preserve admission -> path-mutation ordering. A replacement must + // not change the source between its size check and the hardlink. + let admission = self.capacity_admission_gate.lock().await; + let source_lease = self.begin_capacity_mutation(&root.path, source, key.session_path_component()).await; + let actual_size = fs::metadata(source).await?.len(); + let remaining = { + let capacity = self.capacity.lock_unpoisoned(); + if checks_source_owner && capacity.write_owners.get(source) != Some(&budget.id) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "revision source write replaced")); + } + capacity.revision_reservations.get(&budget.id).map_or(0, |(_, bytes)| *bytes) + }; + drop(admission); + if actual_size <= remaining { + break (actual_size, source_lease); + } + drop(source_lease); + self.ensure_write_budget_available(key, &budget, actual_size).await?; + }; + let budget = Some(budget); let registration = self.register_active_temp_path(&staged_path)?; - if fs::hard_link(source, &staged_path).await.is_ok() { - let size = fs::metadata(&staged_path).await?.len(); - let staged = StagedCacheObject::registered(staged_path, size, Arc::clone(&root.generation), registration); + let staged = + StagedCacheObject::registered(staged_path.clone(), size, Arc::clone(&root.generation), registration) + .with_write_budget(budget.clone())?; + let source_path = source.to_path_buf(); + #[cfg(test)] + let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeRevisionLink); + // A Tokio filesystem call can finish after its caller is cancelled. + // Retain staging and its budget until the link has a definite result. + let (staged, fallback_source) = super::staging::run_owned_cache_operation( + Arc::clone(&self.owned_operation_permits), + "revision-link", + async move { + #[cfg(test)] + if let Some(checkpoint) = checkpoint { + checkpoint.wait().await?; + } + let fallback = if fs::hard_link(&source_path, &staged_path).await.is_ok() { + None + } else { + // Opening under the same source lease also freezes the + // inode used by a copy fallback after replacement resumes. + Some(File::open(source_path).await?) + }; + drop(source_lease); + Ok((staged, fallback)) + }, + ) + .await?; + if let Some(file) = fallback_source { + self.remove_staged(staged).await?; + let staged = self.stage_temp_with_budget(key, file, deadline, budget).await?; self.commit_staged(key, staged).await } else { - drop(registration); - let file = File::open(source).await?; - let staged = self.stage_temp_with_deadline(key, file, deadline).await?; self.commit_staged(key, staged).await } } - /// Revision file bytes as of the last completed capacity scan. + /// Current revision file bytes after completed accounting mutations. pub fn revision_disk_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().revision_bytes } /// Walks the cache tree for the current revision file bytes. diff --git a/backend/hls/src/cache/staging.rs b/backend/hls/src/cache/staging.rs index 70938876d..146f6cbcc 100644 --- a/backend/hls/src/cache/staging.rs +++ b/backend/hls/src/cache/staging.rs @@ -34,7 +34,20 @@ impl StagedCacheObject { cache_path_generation: Arc, registration: Arc, ) -> Self { - Self { path, size, cache_path_generation, _registration: registration } + Self { + cleanup: Arc::new(super::StagedFileCleanup { + path: path.clone(), + registration: Arc::clone(®istration), + budget: None, + completed: std::sync::atomic::AtomicBool::new(false), + writer: std::sync::Mutex::new(None), + }), + path, + size, + cache_path_generation, + _registration: registration, + write_budget: None, + } } } @@ -118,9 +131,10 @@ impl HlsSegmentCache { G: Send + 'static, { let staged_path = staged.path.clone(); + let cleanup = Arc::clone(&staged.cleanup); let result = self.commit_staged_inner(key, staged, guard).await; if result.is_err() { - remove_temp_file_after_error(&staged_path, "commit").await; + cleanup.completed.store(remove_temp_file_after_error(&staged_path, "commit").await, Ordering::Release); } result } @@ -164,13 +178,14 @@ impl HlsSegmentCache { }; fs::create_dir_all(parent).await?; let reservation = self - .prepare_capacity_replacement( + .prepare_capacity_replacement_with_budget( &cache_path.path, &final_path, key.session_path_component(), staged.size, self.max_cache_bytes.load(Ordering::Acquire), self.max_session_bytes.load(Ordering::Acquire), + staged.write_budget.clone(), ) .await; let reservation = match reservation { @@ -265,18 +280,25 @@ impl HlsSegmentCache { drop(admission_gate); let staged_path = staged.path.clone(); let committed_size = staged.size; + #[cfg(test)] + let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeCacheRename); // Once spawned, the owned mutation deliberately outlives cancellation of the requesting future. This // keeps the path lease, temp registration, and accounting reservation alive until the atomic filesystem // operation has a definite result. A runtime shutdown may abort the task, but no cache work can follow // that shutdown; panic/abort drops the reservation and forces accounting rollback/invalidation. return run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "commit", async move { let mut reservation = reservation; + #[cfg(test)] + if let Some(checkpoint) = checkpoint { + checkpoint.wait().await?; + } reservation.mark_filesystem_mutation_started(); if let Err(err) = fs::rename(&staged_path, &final_path).await { remove_temp_file_after_error(&staged_path, "rename").await; return Err(err); } reservation.finish_replacement(); + staged.cleanup.completed.store(true, Ordering::Release); drop(staged); drop(commit_gate); Ok((CachedSegmentMetadata { path: final_path, size: committed_size }, guard)) @@ -286,11 +308,17 @@ impl HlsSegmentCache { } pub async fn remove_staged(&self, staged: StagedCacheObject) -> io::Result<()> { + staged.finish_writer().await; + #[cfg(test)] + self.check_cache_delete_test_failure(&staged.path)?; let result = match fs::remove_file(&staged.path).await { Ok(()) => Ok(()), Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), Err(err) => Err(err), }; + if result.is_ok() { + staged.cleanup.completed.store(true, Ordering::Release); + } drop(staged); result } @@ -315,6 +343,20 @@ impl HlsSegmentCache { reader: &mut R, deadline: Option, ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.stage_temp_inner_with_budget(key, reader, deadline, None).await + } + + pub(super) async fn stage_temp_inner_with_budget( + &self, + key: &K, + reader: &mut R, + deadline: Option, + budget: Option>, + ) -> io::Result where K: HlsCacheObjectKey, R: AsyncRead + Unpin, @@ -322,6 +364,10 @@ impl HlsSegmentCache { if deadline.is_some_and(|deadline| deadline <= Instant::now()) { return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); } + let budget = match budget { + Some(budget) => budget, + None => Arc::new(self.reserve_revision_peak_reclaiming(key, 0).await?), + }; let cache_path = self.cache_path_snapshot(); self.ensure_cache_root_marker_for(&cache_path.path).await?; let final_path = Self::path_for_key_in(&cache_path.path, key); @@ -330,28 +376,61 @@ impl HlsSegmentCache { }; fs::create_dir_all(parent).await?; - let (temp_path, mut temp_file, registration) = self.create_temp_file(key, &cache_path).await?; + let (temp_path, temp_file, registration) = self.create_temp_file(key, &cache_path).await?; + let cleanup = StagedCacheObject::registered( + temp_path.clone(), + 0, + Arc::clone(&cache_path.generation), + Arc::clone(®istration), + ) + .with_write_budget(Some(Arc::clone(&budget)))?; + let writer = cleanup.track_writer(temp_file); if deadline.is_some_and(|deadline| deadline <= Instant::now()) { - drop(temp_file); - remove_temp_file_after_error(&temp_path, "expired-write").await; + cleanup.finish_writer().await; + cleanup + .cleanup + .completed + .store(remove_temp_file_after_error(&temp_path, "expired-write").await, Ordering::Release); return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); } let copy = async { + let mut writer = writer.lock().await; + let temp_file = writer.as_mut().ok_or_else(|| io::Error::other("staging writer already closed"))?; let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); let mut limited = reader.take(max_object_bytes.saturating_add(1)); - let size = tokio::io::copy(&mut limited, &mut temp_file).await?; + let mut size = 0u64; + let mut buffer = [0u8; 16 * 1024]; + loop { + let read = limited.read(&mut buffer).await?; + if read == 0 { + break; + } + let next_size = size.saturating_add(u64::try_from(read).map_err(io::Error::other)?); + if next_size > max_object_bytes { + return Err(cache_object_limit_error(max_object_bytes)); + } + // Admission precedes every filesystem write, including headerless + // bodies and standalone cache users without a startup transaction. + self.ensure_write_budget_available(key, &budget, next_size).await?; + temp_file.write_all(&buffer[..read]).await?; + size = next_size; + } if size > max_object_bytes { return Err(cache_object_limit_error(max_object_bytes)); } temp_file.flush().await?; - drop(temp_file); + writer.take(); Ok::(size) }; let copy_result = if let Some(deadline) = deadline { if let Ok(result) = timeout_at(deadline, copy).await { result } else { - remove_temp_file_after_error(&temp_path, "timed-out-write").await; + cleanup.finish_writer().await; + cleanup + .cleanup + .completed + .store(remove_temp_file_after_error(&temp_path, "timed-out-write").await, Ordering::Release); return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); } } else { @@ -359,10 +438,16 @@ impl HlsSegmentCache { }; match copy_result { Ok(size) => { - Ok(StagedCacheObject::registered(temp_path, size, Arc::clone(&cache_path.generation), registration)) + let mut staged = cleanup; + staged.size = size; + Ok(staged) } Err(err) => { - remove_temp_file_after_error(&temp_path, "failed-write").await; + cleanup.finish_writer().await; + cleanup + .cleanup + .completed + .store(remove_temp_file_after_error(&temp_path, "failed-write").await, Ordering::Release); Err(err) } } @@ -411,12 +496,13 @@ impl HlsSegmentCache { } } -async fn remove_temp_file_after_error(path: &Path, operation: &'static str) { +async fn remove_temp_file_after_error(path: &Path, operation: &'static str) -> bool { match fs::remove_file(path).await { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Ok(()) => true, + Err(error) if error.kind() == io::ErrorKind::NotFound => true, Err(error) => { warn!("HLS cache temporary file cleanup failed: operation={operation} error_kind={:?}", error.kind()); + false } } } diff --git a/backend/hls/src/cache/test_support.rs b/backend/hls/src/cache/test_support.rs new file mode 100644 index 000000000..36051dfe0 --- /dev/null +++ b/backend/hls/src/cache/test_support.rs @@ -0,0 +1,69 @@ +use super::HlsSegmentCache; +use crate::sync_ext::MutexExt; +use std::{ + io, + path::{Path, PathBuf}, +}; +use tokio::sync::oneshot; + +#[derive(Clone, Copy, Eq, PartialEq)] +pub(crate) enum StartupWritePhase { + AfterInventorySnapshot, + BeforeNormalCommit, + AfterNormalCommit, + BeforeRevisionLink, + BeforeCacheRename, + AfterPublication, +} + +pub(super) struct StartupWriteCheckpoint { + phase: StartupWritePhase, + entered: oneshot::Sender<()>, + resume: oneshot::Receiver<()>, +} + +impl StartupWriteCheckpoint { + pub(super) async fn wait(self) -> io::Result<()> { + let _ = self.entered.send(()); + self.resume.await.map_err(|_| io::Error::from_raw_os_error(28)) + } +} + +impl HlsSegmentCache { + pub(crate) fn pause_startup_write(&self, phase: StartupWritePhase) -> (oneshot::Receiver<()>, oneshot::Sender<()>) { + let (entered, reached) = oneshot::channel(); + let (resume, released) = oneshot::channel(); + *self.startup_test_checkpoint.lock_unpoisoned() = + Some(StartupWriteCheckpoint { phase, entered, resume: released }); + (reached, resume) + } + + pub(super) fn take_startup_write_checkpoint(&self, phase: StartupWritePhase) -> Option { + let mut hook = self.startup_test_checkpoint.lock_unpoisoned(); + if hook.as_ref().is_some_and(|hook| hook.phase == phase) { + hook.take() + } else { + None + } + } + + pub(crate) async fn startup_write_checkpoint(&self, phase: StartupWritePhase) -> io::Result<()> { + if let Some(hook) = self.take_startup_write_checkpoint(phase) { + hook.wait().await?; + } + Ok(()) + } + + pub(crate) fn fail_next_cache_deletes(&self, path: PathBuf, attempts: usize) { + self.delete_test_failures.lock_unpoisoned().insert(path, attempts); + } + + pub(super) fn check_cache_delete_test_failure(&self, path: &Path) -> io::Result<()> { + let mut failures = self.delete_test_failures.lock_unpoisoned(); + if let Some(remaining) = failures.get_mut(path).filter(|remaining| **remaining > 0) { + *remaining -= 1; + return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected cache deletion failure")); + } + Ok(()) + } +} diff --git a/backend/hls/src/cache/tests/admission.rs b/backend/hls/src/cache/tests/admission.rs index 9bb0758f7..c7cde451c 100644 --- a/backend/hls/src/cache/tests/admission.rs +++ b/backend/hls/src/cache/tests/admission.rs @@ -42,6 +42,8 @@ async fn abandoned_capacity_reservation_rolls_back_before_filesystem_mutation() session_component: "session".to_string(), replacement: None, filesystem_mutation_started: false, + write_budget: None, + transferred_bytes: 0, capacity: Arc::clone(&cache.capacity), changed: Arc::clone(&cache.capacity_changed), }; @@ -79,6 +81,8 @@ async fn abandoned_started_filesystem_mutation_invalidates_capacity_snapshot() { session_component: "session".to_string(), replacement: None, filesystem_mutation_started: false, + write_budget: None, + transferred_bytes: 0, capacity: Arc::clone(&cache.capacity), changed: Arc::clone(&cache.capacity_changed), }; @@ -131,22 +135,6 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write cache.write_bytes_and_commit(&second_resident, b"0123456789").await.expect("second resident commits"); cache.update_cache_limits(25, 25); - let first_staged = cache - .stage_temp_with_deadline( - &first_target, - &b"abcdefghij"[..], - tokio::time::Instant::now() + Duration::from_secs(5), - ) - .await - .expect("first target stages"); - let second_staged = cache - .stage_temp_with_deadline( - &second_target, - &b"klmnopqrst"[..], - tokio::time::Instant::now() + Duration::from_secs(5), - ) - .await - .expect("second target stages"); let (first_reclaimed_tx, first_reclaimed_rx) = oneshot::channel(); let (resume_first_tx, resume_first_rx) = oneshot::channel(); let reclaimer = Arc::new(PausingCapacityReclaimer { @@ -158,11 +146,13 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write cache.install_capacity_reclaimer(&reclaimer); let first_cache = Arc::clone(&cache); - let first_task = tokio::spawn(async move { first_cache.commit_staged(&first_target, first_staged).await }); + let first_task = + tokio::spawn(async move { first_cache.write_bytes_and_commit(&first_target, b"abcdefghij").await }); first_reclaimed_rx.await.expect("first reclamation pauses after deleting one resident"); let second_cache = Arc::clone(&cache); - let second_task = tokio::spawn(async move { second_cache.commit_staged(&second_target, second_staged).await }); + let second_task = + tokio::spawn(async move { second_cache.write_bytes_and_commit(&second_target, b"klmnopqrst").await }); for _ in 0..100 { tokio::task::yield_now().await; } diff --git a/backend/hls/src/cache/write_budget.rs b/backend/hls/src/cache/write_budget.rs new file mode 100644 index 000000000..3b56b9a6c --- /dev/null +++ b/backend/hls/src/cache/write_budget.rs @@ -0,0 +1,393 @@ +//! Ownership of peak bytes across staging, rename, and immutable publication. +use super::{ + error::capacity_error, revisions::revision_reserved_bytes, ActiveTempFileRegistration, CacheCapacityPressure, + CapacityRevision, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsCacheCapacityRevision, + HlsCacheObjectKey, HlsRevisionDiskReservation, HlsSegmentCache, StagedCacheObject, +}; +use crate::sync_ext::MutexExt; +use std::{ + io, + path::PathBuf, + sync::{atomic::Ordering, Arc}, +}; +use tokio::{ + fs, + io::{AsyncRead, AsyncWriteExt}, + time::Instant, +}; + +type TrackedWriter = Arc>>; + +pub(super) struct StagedFileCleanup { + pub(super) path: PathBuf, + pub(super) registration: Arc, + pub(super) budget: Option>, + pub(super) completed: std::sync::atomic::AtomicBool, + pub(super) writer: std::sync::Mutex>, +} + +impl Drop for StagedFileCleanup { + fn drop(&mut self) { + if self.completed.load(Ordering::Acquire) { + return; + } + let path = self.path.clone(); + let registration = Arc::clone(&self.registration); + let budget = self.budget.clone(); + let writer = self.writer.lock_unpoisoned().clone(); + // Keep both the file registration and peak budget until removal completes. + // A successful rename leaves no file at this unique temporary path. + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(async move { + finish_writer(writer).await; + if let Err(error) = fs::remove_file(path).await { + if error.kind() != io::ErrorKind::NotFound { + log::warn!("HLS staging cleanup deferred: error_kind={:?}", error.kind()); + if let Some(budget) = &budget { + let mut capacity = budget.capacity.lock_unpoisoned(); + capacity.initialized = false; + capacity.revision = Arc::new(CapacityRevision); + } + } + } + drop(registration); + drop(budget); + }); + } + } +} + +async fn finish_writer(writer: Option) { + if let Some(writer) = writer { + let mut guard = writer.lock().await; + if let Some(mut file) = guard.take() { + // Tokio filesystem writes may outlive cancellation of write_all. + // Flushing the same handle joins that pending write before unlink. + let _ = file.flush().await; + } + } +} + +impl StagedCacheObject { + pub(crate) fn track_writer(&self, file: fs::File) -> TrackedWriter { + let writer = Arc::new(tokio::sync::Mutex::new(Some(file))); + *self.cleanup.writer.lock_unpoisoned() = Some(Arc::clone(&writer)); + writer + } + + pub(super) async fn finish_writer(&self) { + let writer = self.cleanup.writer.lock_unpoisoned().clone(); + finish_writer(writer).await; + } + + pub(crate) fn with_write_budget(mut self, budget: Option>) -> io::Result { + Arc::get_mut(&mut self.cleanup) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "cannot change a shared staged file's budget"))? + .budget + .clone_from(&budget); + self.write_budget = budget; + Ok(self) + } +} + +impl HlsSegmentCache { + pub(crate) async fn stage_temp_with_budget( + &self, + key: &K, + mut reader: R, + deadline: Instant, + budget: Option>, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.stage_temp_inner_with_budget(key, &mut reader, Some(deadline), budget).await + } + + pub(crate) async fn ensure_write_budget_available( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + size: u64, + ) -> io::Result<()> { + if !Arc::ptr_eq(&self.capacity, &budget.capacity) + || !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation) + { + return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation")); + } + let remaining = self + .capacity + .lock_unpoisoned() + .revision_reservations + .get(&budget.id) + .filter(|(owner, _)| owner == &key.session_path_component()) + .map(|(_, bytes)| *bytes) + .ok_or_else(|| io::Error::new(io::ErrorKind::Interrupted, "write reservation no longer current"))?; + if size > remaining { + if let Err(error) = self.grow_revision_peak(key, budget, size - remaining).await { + if let Some(capacity) = super::hls_cache_capacity_from_io(&error) { + let mut pressure = capacity.pressure(); + pressure.staged_bytes = size; + return Err(capacity_error( + pressure, + HlsCacheCapacityReclaimOutcome { + protected_working_set_bytes: capacity.protected_working_set_bytes, + reclaimable_bytes: capacity.reclaimable_bytes, + ..Default::default() + }, + capacity.revision().clone(), + )); + } + return Err(error); + } + } + Ok(()) + } + + pub(crate) async fn grow_revision_peak( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + additional: u64, + ) -> io::Result<()> { + let _admission = self.capacity_admission_gate.lock().await; + let root = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&root.path).await?; + let error = match self.try_grow_revision_peak(key, budget, additional) { + Ok(()) => return Ok(()), + Err(error) => error, + }; + let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) }; + let outcome = self + .reclaim_capacity(HlsCacheCapacityReclaimRequest { + proxy_session_id: key.proxy_session_id().clone(), + target_path: self.object_path(key), + required_session_bytes: capacity.required_session_bytes(), + required_global_bytes: capacity.required_global_bytes(), + staged_bytes: additional, + }) + .await?; + self.ensure_capacity_initialized(&root.path).await?; + self.try_grow_revision_peak(key, budget, additional).map_err(|error| { + match super::hls_cache_capacity_from_io(&error) { + Some(capacity) => capacity_error(capacity.pressure(), outcome, capacity.revision().clone()), + None => error, + } + }) + } + + fn try_grow_revision_peak( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + additional: u64, + ) -> io::Result<()> { + if !Arc::ptr_eq(&self.capacity, &budget.capacity) + || !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation) + { + return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired cache write reservation")); + } + let session = key.session_path_component(); + let mut capacity = self.capacity.lock_unpoisoned(); + let (global, local) = revision_reserved_bytes(&capacity, &session); + let global = global.saturating_add(capacity.total_bytes); + let local = local.saturating_add(capacity.session_bytes.get(&session).copied().unwrap_or_default()); + let global_limit = self.max_cache_bytes.load(Ordering::Acquire); + let local_limit = self.max_session_bytes.load(Ordering::Acquire); + if global.saturating_add(additional) > global_limit || local.saturating_add(additional) > local_limit { + return Err(capacity_error( + CacheCapacityPressure { + configured_session_bytes: local_limit, + configured_global_bytes: global_limit, + current_session_bytes: local, + current_global_bytes: global, + staged_bytes: additional, + required_session_bytes: local.saturating_add(additional).saturating_sub(local_limit), + required_global_bytes: global.saturating_add(additional).saturating_sub(global_limit), + }, + HlsCacheCapacityReclaimOutcome::default(), + HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), + )); + } + let (owner, remaining) = capacity + .revision_reservations + .get_mut(&budget.id) + .filter(|(owner, _)| owner == &session) + .ok_or_else(|| io::Error::other("write reservation no longer current"))?; + let _ = owner; + *remaining = remaining.checked_add(additional).ok_or_else(|| io::Error::other("write budget overflow"))?; + capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.capacity_changed.notify_waiters(); + Ok(()) + } + + /// Admission of a complete fresh attempt; unlike replacement admission this + /// cannot credit a leftover stable file against the next attempt's peak. + pub(crate) async fn ensure_revision_peak_capacity( + &self, + key: &K, + peak: u64, + ) -> io::Result<()> { + let reservation = self.reserve_revision_peak_reclaiming(key, peak).await?; + drop(reservation); + Ok(()) + } + + pub(crate) async fn reserve_revision_peak_reclaiming( + &self, + key: &K, + peak: u64, + ) -> io::Result { + // Reserve and reclaim under one admission gate; the reclaimer must never + // reenter admission. It only deletes already-owned files. + let _admission = self.capacity_admission_gate.lock().await; + let mut outcome = HlsCacheCapacityReclaimOutcome::default(); + for attempt in 0..2 { + match self.reserve_revision_peak_inner(key, peak).await { + Ok(reservation) => return Ok(reservation), + Err(error) => { + let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) }; + if attempt == 1 { + return Err(capacity_error(capacity.pressure(), outcome, capacity.revision().clone())); + } + outcome = self + .reclaim_capacity(HlsCacheCapacityReclaimRequest { + proxy_session_id: key.proxy_session_id().clone(), + target_path: self.object_path(key), + required_session_bytes: capacity.required_session_bytes(), + required_global_bytes: capacity.required_global_bytes(), + staged_bytes: peak, + }) + .await?; + } + } + } + unreachable!("bounded admission loop returns on second attempt") + } + + /// Deletes only the stable file installed by this exact peak owner. The + /// path mutation ticket serializes this comparison with every later rename. + pub(crate) async fn delete_write_owned( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + ) -> io::Result<()> { + let path = Self::path_for_key_in(&budget.cache_path, key); + let mutation = self.begin_capacity_mutation(&budget.cache_path, &path, key.session_path_component()).await; + let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&budget.id); + if owned { + let result = self.delete_with_reservation(path.clone(), mutation).await; + if result.is_err() { + self.capacity + .lock_unpoisoned() + .abandoned_writes + .insert(path, (budget.id, key.session_path_component())); + } + result + } else { + Ok(()) + } + } + + pub(crate) async fn retry_abandoned_write_deletions(&self) { + let pending: Vec<_> = self + .capacity + .lock_unpoisoned() + .abandoned_writes + .iter() + .take(super::MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN) + .map(|(path, owner)| (path.clone(), owner.clone())) + .collect(); + let root = self.cache_path_snapshot(); + for (path, (owner, session)) in pending { + if !path.starts_with(&root.path) { + self.capacity.lock_unpoisoned().abandoned_writes.remove(&path); + continue; + } + let mutation = self.begin_capacity_mutation(&root.path, &path, session).await; + let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&owner); + if owned { + let _ = self.delete_with_reservation(path, mutation).await; + } else { + self.capacity.lock_unpoisoned().abandoned_writes.remove(&path); + } + } + } + + pub(crate) fn register_reserved_spool(&self, path: PathBuf, budget: &HlsRevisionDiskReservation) { + let mut capacity = self.capacity.lock_unpoisoned(); + capacity.write_owners.insert(path.clone(), budget.id); + capacity.reserved_spools.insert(path, budget.id); + } + + pub(crate) fn account_reserved_spool( + &self, + path: &std::path::Path, + size: u64, + budget: &HlsRevisionDiskReservation, + ) -> io::Result<()> { + let mut capacity = self.capacity.lock_unpoisoned(); + let (session, remaining) = capacity + .revision_reservations + .get_mut(&budget.id) + .ok_or_else(|| io::Error::other("spool reservation expired"))?; + let session = session.clone(); + *remaining = remaining.checked_sub(size).ok_or_else(|| io::Error::other("spool exceeds peak budget"))?; + capacity.reserved_spools.remove(path); + capacity.total_bytes = capacity.total_bytes.saturating_add(size); + *capacity.session_bytes.entry(session).or_default() += size; + capacity.revision_bytes = capacity.revision_bytes.saturating_add(size); + super::capacity::update_revision_metrics(&capacity); + capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.capacity_changed.notify_waiters(); + Ok(()) + } + + pub(crate) fn install_revision_metrics(&self, metrics: &Arc) { + let mut capacity = self.capacity.lock_unpoisoned(); + capacity.revision_metrics = Some(Arc::downgrade(metrics)); + super::capacity::update_revision_metrics(&capacity); + } + + pub(crate) fn accounted_session_bytes(&self, session: &crate::ProxySessionId) -> u64 { + let key = super::SegmentCacheKey::new(session.clone(), 0, "ts"); + self.capacity.lock_unpoisoned().session_bytes.get(&key.session_path_component()).copied().unwrap_or_default() + } + + pub(crate) fn accounted_global_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().total_bytes } + + pub(crate) async fn session_revision_disk_bytes(&self, session: &crate::ProxySessionId) -> io::Result { + let key = super::SegmentCacheKey::new(session.clone(), 0, "ts"); + let mut entries = match fs::read_dir(self.cache_path_snapshot().path.join(key.session_path_component())).await { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(0), + Err(error) => return Err(error), + }; + let mut bytes = 0u64; + while let Some(entry) = entries.next_entry().await? { + if super::revisions::is_revision_cache_path(&entry.path()) && entry.file_type().await?.is_file() { + match entry.metadata().await { + Ok(metadata) => bytes = bytes.saturating_add(metadata.len()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + } + Ok(bytes) + } + + pub(crate) fn remaining_revision_bytes(&self, session: &crate::ProxySessionId) -> u64 { + let key = super::SegmentCacheKey::new(session.clone(), 0, "ts"); + revision_reserved_bytes(&self.capacity.lock_unpoisoned(), &key.session_path_component()).1 + } +} + +impl HlsRevisionDiskReservation { + /// Resolves cleanup against this write's original root, even after handoff. + pub(crate) fn object_path(&self, key: &K) -> PathBuf { + HlsSegmentCache::path_for_key_in(&self.cache_path, key) + } +} diff --git a/backend/hls/src/gc.rs b/backend/hls/src/gc.rs index 76eca2369..876c8775e 100644 --- a/backend/hls/src/gc.rs +++ b/backend/hls/src/gc.rs @@ -37,6 +37,7 @@ pub struct HlsGarbageCollector { pending_cache_deletions: Arc>, access_leases: StdRwLock>>>, run_once_gate: AsyncMutex<()>, + revision_store: StdRwLock>>, } #[derive(Debug, Clone, Eq, PartialEq)] @@ -112,6 +113,7 @@ mod deletion; mod policy; mod reclaim; mod report; +mod revisions; mod selection; pub use collector::{build_rewrite_secret_fingerprint, exec_hls_cache_gc}; use deletion::{CacheDeletionQueueGeneration, CacheObjectDeletion}; diff --git a/backend/hls/src/gc/collector.rs b/backend/hls/src/gc/collector.rs index 1ab3a2287..36b98299b 100644 --- a/backend/hls/src/gc/collector.rs +++ b/backend/hls/src/gc/collector.rs @@ -52,6 +52,7 @@ impl HlsGarbageCollector { pending_cache_deletions: Arc::new(StdMutex::new(CacheDeletionQueueState::default())), access_leases: StdRwLock::new(None), run_once_gate: AsyncMutex::new(()), + revision_store: StdRwLock::new(None), } } @@ -102,6 +103,11 @@ impl HlsGarbageCollector { report.temp_files_deleted = self.cache.delete_temp_files_older_than(cutoff).await?; let sessions = self.sessions.list_sessions().await; + if let Some(handle) = sessions.first() { + let id = handle.read().await.proxy_session_id.clone(); + self.cache.capacity_usage(&id).await?; + } + self.cleanup_revisions(&sessions, &mut report).await?; let mut active_session_ids = HashSet::with_capacity(sessions.len()); for session in &sessions { active_session_ids.insert(session.read().await.proxy_session_id.clone()); @@ -122,6 +128,7 @@ impl HlsGarbageCollector { session_deletions.persist(&mut report); } + self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await; let mut global_deletions = self.reserve_cache_deletion_batch(); self.collect_global_size_deletions(&sessions, &policy, &mut report, &mut global_deletions).await; global_deletions.persist(&mut report); @@ -131,6 +138,7 @@ impl HlsGarbageCollector { self.remove_idle_session_if_still_idle(session, now_ms, &policy, &mut report).await; } + self.cleanup_revisions(&sessions, &mut report).await?; report.cache_object_deletions_deferred = self.pending_cache_deletion_count(); self.record_report_metrics(&report); if report.did_cleanup_or_invalidate() { diff --git a/backend/hls/src/gc/reclaim.rs b/backend/hls/src/gc/reclaim.rs index ae07f378c..c3f38164e 100644 --- a/backend/hls/src/gc/reclaim.rs +++ b/backend/hls/src/gc/reclaim.rs @@ -4,8 +4,8 @@ use super::{ selection::{ capacity_reclamation_evidence, collect_projected_session_reclamation, duration_expired_head_segment, fifo_head_size_candidate, oldest_global_fifo_head_candidate, oldest_global_transient_object_candidate, - remove_map_entry, remove_segment_entry, remove_stale_queue_entries, session_cache_size, - take_expired_transient_objects, total_sessions_cache_size, unprotected_unreferenced_map_ids, + remove_map_entry, remove_obsolete_deferred_heads, remove_segment_entry, remove_stale_queue_entries, + session_cache_size, take_expired_transient_objects, unprotected_unreferenced_map_ids, }, CacheDeletionBatch, GarbageCollectionPolicy, GarbageCollectionReport, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle, @@ -41,6 +41,8 @@ impl HlsGarbageCollector { self.retry_pending_cache_deletions(&mut report, &mut pending_attempt_budget).await; deletion_attempt_budget = deletion_attempt_budget.saturating_sub(pending_attempts_before.saturating_sub(pending_attempt_budget)); + let sessions = self.sessions.list_sessions().await; + self.cleanup_revisions(&sessions, &mut report).await?; let after_pending = self.cache.capacity_usage(&request.proxy_session_id).await?; let required_session_bytes = request .required_session_bytes @@ -73,6 +75,30 @@ impl HlsGarbageCollector { Some(&request.target_path), release_through, ); + let head = session.segments.first_key_value(); + let reason = match head { + None => "no-candidates", + Some((seq, _)) if protected.segment_proxy_seqs.contains(seq) => "protected", + Some((_, entry)) => match entry.status { + super::SegmentCacheStatus::Ready { .. } => "reclaimable", + super::SegmentCacheStatus::CapacityDeferred { .. } if entry.origin_fetch_ref.is_some() => { + "deferred-origin-binding-retained" + } + super::SegmentCacheStatus::CapacityDeferred { .. } => "obsolete-deferred-head", + super::SegmentCacheStatus::Discovered => "not-cached", + super::SegmentCacheStatus::Queued { .. } => "queued", + super::SegmentCacheStatus::Fetching { .. } => "active-fetch", + super::SegmentCacheStatus::FailedRetryable { .. } + | super::SegmentCacheStatus::FailedPermanent { .. } => "failed-retention", + super::SegmentCacheStatus::Expired => "expired-retention", + }, + }; + log::debug!( + "HLS capacity FIFO: proxy_session={} head={:?} status={:?} blocker={reason}", + crate::safe_proxy_session_id(&session.proxy_session_id), + head.map(|(seq, _)| *seq), + head.map(|(_, entry)| &entry.status) + ); let evidence = capacity_reclamation_evidence(&session, &protected); protected_working_set_bytes = evidence.protected_working_set_bytes; reclaimable_bytes = evidence.reclaimable_bytes; @@ -95,8 +121,8 @@ impl HlsGarbageCollector { let _ = session.render_and_store_manifest(rendered_at_ms); } } else { - protected_working_set_bytes = after_pending.session_bytes; session_lock_busy = true; + log::debug!("HLS capacity selection: proxy_session={} outcome=lock-conflict protected_working_set_known=false", crate::safe_proxy_session_id(&request.proxy_session_id)); } drop(lease_guard); if session_lock_busy { @@ -112,22 +138,15 @@ impl HlsGarbageCollector { } } + self.cleanup_revisions(&sessions, &mut report).await?; let after_session = self.cache.capacity_usage(&request.proxy_session_id).await?; let remaining_session_bytes = request .required_session_bytes .saturating_sub(before.session_bytes.saturating_sub(after_session.session_bytes)); - if remaining_session_bytes > 0 { - return Ok(HlsCacheCapacityReclaimOutcome { - reclaimed_session_bytes: before.session_bytes.saturating_sub(after_session.session_bytes), - reclaimed_global_bytes: before.global_bytes.saturating_sub(after_session.global_bytes), - protected_working_set_bytes, - reclaimable_bytes, - }); - } let required_global_bytes = request .required_global_bytes .saturating_sub(before.global_bytes.saturating_sub(after_session.global_bytes)); - if required_global_bytes > 0 { + if remaining_session_bytes == 0 && required_global_bytes > 0 { let sessions = self.sessions.list_sessions().await; let mut deletions = self.reserve_cache_deletion_batch(); self.collect_projected_global_reclamation( @@ -139,9 +158,13 @@ impl HlsGarbageCollector { ) .await; deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await; + self.cleanup_revisions(&sessions, &mut report).await?; } let after = self.cache.capacity_usage(&request.proxy_session_id).await?; + if log::log_enabled!(log::Level::Debug) { + self.log_capacity_inventory(&request.proxy_session_id, &report).await; + } Ok(HlsCacheCapacityReclaimOutcome { reclaimed_session_bytes: before.session_bytes.saturating_sub(after.session_bytes), reclaimed_global_bytes: before.global_bytes.saturating_sub(after.global_bytes), @@ -150,6 +173,25 @@ impl HlsGarbageCollector { }) } + /// Diagnostic I/O cannot change the result of capacity reclamation. + pub(super) async fn log_capacity_inventory(&self, id: &crate::ProxySessionId, report: &GarbageCollectionReport) { + let revision_bytes = match self.cache.session_revision_disk_bytes(id).await { + Ok(bytes) => bytes, + Err(error) => { + log::debug!( + "HLS capacity inventory unavailable: proxy_session={} error_kind={:?}", + crate::safe_proxy_session_id(id), + error.kind() + ); + return; + } + }; + let reserved_bytes = self.cache.remaining_revision_bytes(id); + log::debug!("HLS capacity inventory: proxy_session={} revision_bytes={revision_bytes} remaining_reserved_bytes={reserved_bytes} revision_deletes_planned={} revision_deletes_succeeded={} revision_deletes_failed={} revision_reclaimed_bytes={}", + crate::safe_proxy_session_id(id), report.revision_deletions_planned, + report.revision_deletions_succeeded, report.revision_deletions_failed, report.revision_bytes_reclaimed); + } + pub(super) fn collect_session_deletions( session: &mut HlsSession, cache: &HlsSegmentCache, @@ -161,6 +203,7 @@ impl HlsGarbageCollector { report.stale_queue_entries_removed = report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session)); + remove_obsolete_deferred_heads(session, cache, None, None, deletions); let transient_before = session.transient.resources.len(); let protected = protected_set_for_cache_reclamation(session, cache, None, None); session.transient.prune_expired_except(now_ms, &protected.key_resource_ids); @@ -195,7 +238,8 @@ impl HlsGarbageCollector { } } - let mut session_size = session_cache_size(session); + let mut session_size = + cache.accounted_session_bytes(&session.proxy_session_id).max(session_cache_size(session)); while session_size > policy.cache_bytes_per_session && deletions.has_capacity() { let protected = protected_set_for_cache_reclamation(session, cache, None, None); let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) @@ -247,7 +291,7 @@ impl HlsGarbageCollector { report: &mut GarbageCollectionReport, deletions: &mut CacheDeletionBatch, ) { - let mut total_size = total_sessions_cache_size(sessions).await; + let mut total_size = self.cache.accounted_global_bytes(); let mut transient_readiness_advanced = HashSet::new(); let mut skipped_transient_sessions = HashSet::new(); diff --git a/backend/hls/src/gc/report.rs b/backend/hls/src/gc/report.rs index 261ce8660..0018f2579 100644 --- a/backend/hls/src/gc/report.rs +++ b/backend/hls/src/gc/report.rs @@ -2,6 +2,11 @@ use super::ProxySessionId; #[derive(Debug, Default, Clone, Eq, PartialEq)] pub struct GarbageCollectionReport { + pub revision_deletions_planned: usize, + pub revision_deletions_succeeded: usize, + pub revision_deletions_failed: usize, + pub revision_bytes_reclaimed: u64, + pub revision_lock_conflicts: usize, pub secret_cache_invalidated: bool, pub secret_cache_invalidation_deferred: bool, pub temp_files_deleted: usize, @@ -29,7 +34,9 @@ impl GarbageCollectionReport { } pub fn did_cleanup_or_invalidate(&self) -> bool { - self.secret_cache_invalidated + self.revision_deletions_planned > 0 + || self.revision_deletions_failed > 0 + || self.secret_cache_invalidated || self.secret_cache_invalidation_deferred || self.temp_files_deleted > 0 || self.orphan_session_dirs_deleted > 0 diff --git a/backend/hls/src/gc/revisions.rs b/backend/hls/src/gc/revisions.rs new file mode 100644 index 000000000..c8e4d7aa3 --- /dev/null +++ b/backend/hls/src/gc/revisions.rs @@ -0,0 +1,141 @@ +//! Shared, bounded retirement for regular GC and projected-capacity reclamation. +use super::{GarbageCollectionReport, HlsGarbageCollector, HlsSessionHandle}; +use crate::{sync_ext::MutexExt, SegmentRevision, SegmentRevisionStore}; +use std::{ + collections::VecDeque, + io, + sync::{Arc, Weak}, +}; + +struct RetirementBatch { + store: Arc, + pending: VecDeque>, +} + +impl Drop for RetirementBatch { + fn drop(&mut self) { + // Cancellation or a failed delete preserves retry ownership, including + // all later entries in the batch which have not reached the filesystem. + for revision in self.pending.drain(..) { + let _ = self.store.requeue_retired(revision); + } + } +} + +impl HlsGarbageCollector { + pub(crate) fn install_revision_store(&self, store: &Arc) { + *self.revision_store.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::downgrade(store)); + } + + /// Caller already owns the GC gate. Never calls admission or reacquires GC. + #[allow( + clippy::too_many_lines, + reason = "One bounded lease/session snapshot followed by cancellation-safe filesystem retirement." + )] + pub(super) async fn cleanup_revisions( + &self, + sessions: &[HlsSessionHandle], + report: &mut GarbageCollectionReport, + ) -> io::Result<()> { + self.cache.retry_abandoned_write_deletions().await; + let before = ( + report.revision_deletions_planned, + report.revision_deletions_succeeded, + report.revision_deletions_failed, + report.revision_bytes_reclaimed, + ); + let mut stores: Vec<_> = self + .revision_store + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(Weak::upgrade) + .into_iter() + .collect(); + let leases = self + .access_leases + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(Weak::upgrade); + for handle in sessions { + let mut lease_guard = match &leases { + Some(leases) => { + if let Ok(guard) = leases.try_write() { + Some(guard) + } else { + report.revision_lock_conflicts += 1; + continue; + } + } + None => None, + }; + let Ok(mut session) = handle.try_write() else { + report.revision_lock_conflicts += 1; + continue; + }; + let id = session.proxy_session_id.clone(); + let floor = session.segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq); + let crate::HlsSession { startup, segments, .. } = &mut *session; + if let Some(startup) = startup { + startup.revisions.retain(|seq, _| segments.contains_key(seq)); + if !stores.iter().any(|store| Arc::ptr_eq(store, &startup.store)) { + stores.push(Arc::clone(&startup.store)); + } + } + if let Some(leases) = &mut lease_guard { + leases.prune_revision_bindings(&id, floor); + } + } + let mut remaining = super::MAX_CACHE_DELETE_RETRIES_PER_RUN; + for store in stores { + if remaining == 0 { + break; + } + let pending = store.retire_unpinned_limit(remaining)?; + let mut batch = RetirementBatch { store, pending: pending.into() }; + let attempts = batch.pending.len(); + for _ in 0..attempts { + let Some(revision) = batch.pending.front().cloned() else { break }; + report.revision_deletions_planned += 1; + remaining -= 1; + revision.file_pin.lock_unpoisoned().take(); + let size = self.cache.metadata(&revision.key).await?.map_or(0, |metadata| metadata.size); + match self.cache.delete_if_inactive(&revision.key).await { + Ok(()) => { + batch.pending.pop_front(); + report.revision_deletions_succeeded += 1; + report.revision_bytes_reclaimed = report.revision_bytes_reclaimed.saturating_add(size); + } + Err(error) => { + report.revision_deletions_failed += 1; + log::warn!( + "HLS revision delete deferred: proxy_session={} proxy_seq={} error_kind={:?}", + crate::safe_proxy_session_id(crate::HlsCacheObjectKey::proxy_session_id(&revision.key)), + revision.key.proxy_seq, + error.kind() + ); + // Put failed entries at the end of this bounded batch; + // its Drop requeues them after successful work finishes. + batch.pending.rotate_left(1); + } + } + } + } + self.metrics.record_revision_gc( + report.revision_deletions_planned - before.0, + report.revision_deletions_succeeded - before.1, + report.revision_deletions_failed - before.2, + report.revision_bytes_reclaimed - before.3, + ); + log::debug!( + "HLS revision GC: planned={} succeeded={} failed={} reclaimed_bytes={} lock_conflicts={}", + report.revision_deletions_planned, + report.revision_deletions_succeeded, + report.revision_deletions_failed, + report.revision_bytes_reclaimed, + report.revision_lock_conflicts + ); + Ok(()) + } +} diff --git a/backend/hls/src/gc/selection.rs b/backend/hls/src/gc/selection.rs index 4c694e8de..43ee9f261 100644 --- a/backend/hls/src/gc/selection.rs +++ b/backend/hls/src/gc/selection.rs @@ -8,15 +8,6 @@ use super::{ }; use std::{collections::HashSet, sync::Arc}; -pub(super) async fn total_sessions_cache_size(sessions: &[HlsSessionHandle]) -> u64 { - let mut total_size = 0_u64; - for session in sessions { - let session = session.read().await; - total_size = total_size.saturating_add(session_cache_size(&session)); - } - total_size -} - pub(super) fn collect_projected_session_reclamation( session: &mut HlsSession, cache: &HlsSegmentCache, @@ -26,6 +17,7 @@ pub(super) fn collect_projected_session_reclamation( max_deletions: usize, deletions: &mut CacheDeletionBatch, ) { + remove_obsolete_deferred_heads(session, cache, Some(excluded_path), release_through, deletions); let mut planned_bytes = 0_u64; let mut planned_deletions = 0_usize; let mut transient_readiness_changed = false; @@ -335,3 +327,27 @@ pub(super) fn remove_map_entry(session: &mut HlsSession, map_id: ProxyMapId) -> session.origin_map_to_proxy.retain(|_, mapped_map_id| *mapped_map_id != map_id); Some(MapEntryDeletion { key: map.cache_key, content_length }) } + +/// Only retire an inactive deferred FIFO head once its origin binding is gone +/// and every existing working-set protection has released it. +pub(super) fn remove_obsolete_deferred_heads( + session: &mut HlsSession, + cache: &HlsSegmentCache, + excluded: Option<&std::path::Path>, + release_through: Option, + deletions: &mut CacheDeletionBatch, +) { + while deletions.has_capacity() { + let Some((&seq, entry)) = session.segments.first_key_value() else { break }; + let protected = protected_set_for_cache_reclamation(session, cache, excluded, release_through); + if entry.origin_fetch_ref.is_some() + || !matches!(entry.status, SegmentCacheStatus::CapacityDeferred { .. }) + || protected.segment_proxy_seqs.contains(&seq) + { + break; + } + if let Some(key) = remove_segment_entry(session, seq) { + deletions.push(CacheObjectDeletion::Segment { key, reason: SegmentCacheDeletionReason::Duration }); + } + } +} diff --git a/backend/hls/src/gc/tests.rs b/backend/hls/src/gc/tests.rs index 45ddd1f92..3aac8e52f 100644 --- a/backend/hls/src/gc/tests.rs +++ b/backend/hls/src/gc/tests.rs @@ -251,5 +251,6 @@ mod playlist; mod policy; mod publication; mod recovery; +mod revisions; mod streaming; mod transport; diff --git a/backend/hls/src/gc/tests/publication.rs b/backend/hls/src/gc/tests/publication.rs index 8f23ac22c..051327940 100644 --- a/backend/hls/src/gc/tests/publication.rs +++ b/backend/hls/src/gc/tests/publication.rs @@ -191,3 +191,104 @@ async fn transient_object_gc_deletes_the_physical_fill_generation_instead_of_the assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("physical metadata read").is_none()); assert!(gc.cache.metadata(&lookup_key).await.expect("logical metadata read").is_some()); } + +#[test] +fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { + use crate::{ + media_reserve::{ + evaluate_lease_reserve, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackCursor, + HlsLeaseReserveInput, HlsManifestCommitIdentity, HlsManifestDeliveryMode, + }, + recovery_timing::{HlsLeaseCutoverTiming, HlsRecoveryTriggerBudgetMs}, + terminal_tail::HlsMediaContainer, + }; + use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + + let mut session = HlsSession::new(crate::HlsSessionKey::new(1, "gc-probe"), b"probe", 1_000); + let mut body = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:0\n"); + for seq in 0..9 { + write!(body, "#EXTINF:12.0,\n{seq}.ts\n").expect("test manifest writes"); + } + let OriginManifestParseOutcome::Normal(origin) = + parse_origin_media_manifest(&body, "http://origin.example/index.m3u8") + else { + panic!("normal manifest") + }; + session.apply_origin_manifest(&origin).unwrap(); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 }; + } + session.publishable_origin_head_proxy_seq = Some(4); + let mut cursor = HlsLeasePlaybackCursor::default(); + for seq in 0..8 { + let token = cursor.record_request_started(seq, 1_000 + seq * 10_000); + let token = cursor.bind_request_timing(token, &session.segments[&seq]); + let _ = cursor.record_request_completed(token, 1_000 + seq * 10_000); + } + let manifest = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 71_000, + first_proxy_seq: 4, + last_proxy_seq: 6, + visible_segments: (4..7) + .map(|seq| HlsLeaseManifestSegment { + proxy_seq: seq, + duration_ms: 12_000, + uri: format!("{seq}.ts").into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }) + .collect::>() + .into(), + discontinuity_sequence: 0, + target_duration_ms: 12_000, + playlist_duration_ms: 36_000, + last_visible_media_end_ms: 36_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }; + let evaluate = |session: &HlsSession| { + let timeline = session.ready_timeline_snapshot(cursor.ready_timeline_start_proxy_seq(4), 71_000); + let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { + manifest: &manifest, + cursor: &cursor, + ready_timeline: &timeline, + now_ms: 71_000, + playback_rate_guard_milli: 1_050, + recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000), + origin_path_degraded: true, + recovery_committed: false, + }); + (timeline.units.len(), reserve) + }; + let (before_units, before) = evaluate(&session); + assert_eq!(before.guaranteed_reserve_ms, 34_500); + assert!(!ProtectedSet::from_session(&session).segment_proxy_seqs.contains(&0)); + // Run the actual metadata-removal function used by garbage collection. + assert!(remove_segment_entry(&mut session, 0).is_some()); + let (after_units, after) = evaluate(&session); + let still_ready = session.ready_timeline_snapshot(4, 71_000); + assert_eq!(still_ready.units.len(), 5); + assert_eq!(cursor.first_requested_proxy_seq, Some(0)); + assert_eq!(before_units, 5); + assert_eq!(after_units, 5); + assert_eq!(after, before); + let timing = + HlsLeaseCutoverTiming::from_reserve(71_000, after.guaranteed_reserve_ms, after.transition_margin, None); + assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 93_500); + // Removing the entire historical prefix must keep exactly the same media clock. + for seq in 1..4 { + assert!(remove_segment_entry(&mut session, seq).is_some()); + } + assert_eq!(evaluate(&session).1, before); + // A real hole in the current READY span must still shorten the reserve. + assert!(remove_segment_entry(&mut session, 7).is_some()); + assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500); +} diff --git a/backend/hls/src/gc/tests/revisions.rs b/backend/hls/src/gc/tests/revisions.rs new file mode 100644 index 000000000..763e942a8 --- /dev/null +++ b/backend/hls/src/gc/tests/revisions.rs @@ -0,0 +1,248 @@ +use super::*; +use crate::{SegmentRevisionKind, SegmentRevisionStore}; + +#[tokio::test] +async fn diagnostic_inventory_errors_do_not_fail_capacity_reclamation() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let key = crate::SegmentCacheKey::new(id.clone(), 1, "ts"); + assert_eq!(gc.cache.capacity_usage(&id).await?.global_bytes, 0); + tokio::fs::write(gc.cache.object_path(&key).parent().unwrap(), b"not a directory").await?; + assert!(gc.cache.session_revision_disk_bytes(&id).await.is_err()); + gc.log_capacity_inventory(&id, &GarbageCollectionReport::default()).await; + let outcome = gc + .reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest { + proxy_session_id: id.clone(), + target_path: gc.cache.object_path(&key), + required_session_bytes: 0, + required_global_bytes: 0, + staged_bytes: 1, + }) + .await?; + assert_eq!(outcome, crate::HlsCacheCapacityReclaimOutcome::default()); + let occupied = crate::SegmentCacheKey::new(ProxySessionId("occupied".into()), 1, "ts"); + gc.cache.write_bytes_and_commit(&occupied, &[1; 20]).await?; + gc.cache.update_cache_limits(30, 30); + let error = gc.cache.reserve_revision_peak_reclaiming(&key, 20).await.err().unwrap(); + assert!( + crate::cache::hls_cache_capacity_from_io(&error).is_some(), + "diagnostic I/O must preserve the functional capacity error" + ); + assert_eq!(gc.cache.remaining_revision_bytes(&id), 0); + Ok(()) +} + +#[tokio::test] +async fn global_reclamation_retires_revisions_unpinned_by_another_sessions_segment_deletion() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, old_session) = gc_with_session(&directory).await; + let incoming = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "incoming"), b"secret", 1).await; + let old_id = old_session.read().await.proxy_session_id.clone(); + let incoming_id = incoming.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let revision = store.create(old_id.clone(), 1, SegmentRevisionKind::Processed)?; + let revision_key = revision.revision().key.clone(); + let metadata = gc.cache.write_bytes_and_commit(&revision_key, &[7; 20]).await?; + revision.revision().complete(metadata); + *revision.revision().file_pin.lock().unwrap() = + Some(gc.cache.pin_revision_file(&gc.cache.object_path(&revision_key))?); + let old_key = { + let mut session = old_session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.publishable_origin_head_proxy_seq = Some(2); + let head = session.segments.get_mut(&1).unwrap(); + head.origin_fetch_ref = None; + head.status = SegmentCacheStatus::Ready { content_length: 20, ready_at_ms: 0 }; + let key = head.cache_key.clone(); + let config = tuliprox_core::model::HlsStartupConfig::default(); + session.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 1_000, + worker: std::sync::Weak::new(), + access_leases: Arc::new(RwLock::new(HlsAccessLeaseStore::default())), + revisions: [(1, revision)].into(), + policy_fixed: true, + store, + budget: crate::ProgressiveBudgetManager::new(config), + }); + key + }; + gc.cache.write_bytes_and_commit(&old_key, &[1; 20]).await?; + gc.cache.update_cache_limits(50, 50); + assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40); + let key = crate::SegmentCacheKey::new(incoming_id.clone(), 1, "ts"); + let peak = Arc::new( + tokio::time::timeout(Duration::from_secs(2), gc.cache.reserve_revision_peak_reclaiming(&key, 40)) + .await + .expect("global reclamation must finish without a periodic GC")?, + ); + assert!(gc.cache.metadata(&old_key).await?.is_none()); + assert!(gc.cache.metadata(&revision_key).await?.is_none()); + assert_eq!(gc.cache.capacity_usage(&old_id).await?.session_bytes, 0); + assert!(old_session.read().await.startup.as_ref().unwrap().revisions.is_empty()); + let staged = gc + .cache + .stage_temp_with_budget( + &key, + &[9; 40][..], + tokio::time::Instant::now() + Duration::from_secs(2), + Some(Arc::clone(&peak)), + ) + .await?; + gc.cache.commit_staged(&key, staged).await?; + drop(peak); + assert_eq!(gc.cache.remaining_revision_bytes(&incoming_id), 0); + assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40); + assert_eq!(tokio::fs::read(gc.cache.object_path(&key)).await?, [9; 40]); + Ok(()) +} + +#[tokio::test] +async fn targeted_reclamation_deletes_only_unpinned_revisions_of_the_same_session() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let metrics = Arc::new(crate::HlsCacheMetrics::default()); + gc.cache.install_revision_metrics(&metrics); + let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?; + gc.cache.write_bytes_and_commit(&revision.revision().key, &[7; 80]).await?; + let reader = revision.clone(); + drop(revision); + gc.cache.update_cache_limits(100, 100); + let next = crate::SegmentCacheKey::new(id.clone(), 1, "ts"); + assert!(gc.cache.reserve_revision_peak_reclaiming(&next, 30).await.is_err()); + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80); + drop(reader); + let peak = gc.cache.reserve_revision_peak_reclaiming(&next, 30).await?; + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0); + assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?); + assert_eq!(metrics.snapshot().revision_disk_bytes, 0); + drop(peak); + assert_eq!(gc.cache.remaining_revision_bytes(&id), 0); + assert_eq!(gc.run_once(0).await?.revision_deletions_succeeded, 0); + Ok(()) +} + +#[tokio::test] +async fn failed_revision_deletes_preserve_bytes_and_are_retried_idempotently() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?; + let key = revision.revision().key.clone(); + gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?; + gc.cache.fail_next_cache_deletes(gc.cache.object_path(&key), 2); + drop(revision); + let first = gc.run_once(0).await?; + assert_eq!(first.revision_deletions_failed, 2); + assert_eq!(first.revision_bytes_reclaimed, 0); + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80); + assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?); + let next = gc.run_once(0).await?; + assert_eq!(next.revision_deletions_succeeded, 1); + assert_eq!(next.revision_bytes_reclaimed, 80); + assert!(!tokio::fs::try_exists(gc.cache.object_path(&key)).await?); + assert_eq!(gc.run_once(0).await?.revision_deletions_planned, 0); + Ok(()) +} + +#[tokio::test] +async fn obsolete_deferred_fifo_head_retires_without_skipping_a_protected_head() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let head = session.segments.get_mut(&1).unwrap(); + head.status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 1 }; + head.origin_fetch_ref = None; + head.access.reader_started(1); + head.cache_key.clone() + }; + gc.cache.write_bytes_and_commit(&key, b"partial-commit").await?; + let first = gc.run_once(0).await?; + assert_eq!(first.segments_deleted(), 0); + assert!(session.read().await.segments.contains_key(&1)); + session.write().await.segments.get_mut(&1).unwrap().access.reader_finished(); + let second = gc.run_once(0).await?; + assert_eq!(second.segments_deleted(), 1); + assert!(!session.read().await.segments.contains_key(&1)); + assert!(gc.cache.metadata(&key).await?.is_none()); + assert_eq!(gc.run_once(0).await?.segments_deleted(), 0); + Ok(()) +} + +#[tokio::test] +async fn revision_cleanup_skips_contended_locks_and_pressure_keeps_protection_unknown() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let leases = Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default())); + gc.install_access_leases(&leases); + let lease_guard = leases.write().await; + let mut report = GarbageCollectionReport::default(); + tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report)) + .await + .expect("revision cleanup must not wait while the lease lock is held")?; + assert_eq!(report.revision_lock_conflicts, 1); + drop(lease_guard); + + let key = crate::SegmentCacheKey::new(id.clone(), 0, "ts"); + gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?; + let session_guard = session.write().await; + let mut report = GarbageCollectionReport::default(); + tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report)) + .await + .expect("revision cleanup must not reverse the lease/session lock order")?; + assert_eq!(report.revision_lock_conflicts, 1); + let outcome = tokio::time::timeout( + Duration::from_secs(2), + gc.reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest { + proxy_session_id: id.clone(), + target_path: gc.cache.object_path(&key), + required_session_bytes: 1, + required_global_bytes: 0, + staged_bytes: 1, + }), + ) + .await + .expect("targeted selection must not wait for a contended session")?; + assert_eq!(outcome.protected_working_set_bytes, 0, "a lock conflict cannot claim all 80 bytes are protected"); + assert_eq!(outcome.reclaimed_session_bytes, 0); + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80); + drop(session_guard); + Ok(()) +} + +#[tokio::test] +async fn shared_revision_cleanup_retires_a_bounded_batch_and_drains_idempotently() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let limit = super::super::MAX_CACHE_DELETE_RETRIES_PER_RUN; + for seq in 0..limit + 3 { + let revision = store.create(id.clone(), u64::try_from(seq).unwrap(), SegmentRevisionKind::Processed)?; + gc.cache.write_bytes_and_commit(&revision.revision().key, &[7]).await?; + } + for expected in [limit, 3, 0] { + let mut report = GarbageCollectionReport::default(); + gc.cleanup_revisions(&[Arc::clone(&session)], &mut report).await?; + assert_eq!(report.revision_deletions_planned, expected); + assert_eq!(report.revision_deletions_succeeded, expected); + assert_eq!(report.revision_deletions_failed, 0); + assert_eq!(report.revision_bytes_reclaimed, u64::try_from(expected).unwrap()); + assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?); + } + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0); + assert_eq!(gc.cache.remaining_revision_bytes(&id), 0); + Ok(()) +} diff --git a/backend/hls/src/gc/tests/transport.rs b/backend/hls/src/gc/tests/transport.rs index 4dd4c2d0a..49565d70e 100644 --- a/backend/hls/src/gc/tests/transport.rs +++ b/backend/hls/src/gc/tests/transport.rs @@ -68,8 +68,12 @@ async fn global_size_gc_subtracts_map_bytes_after_segment_removal() { session.apply_origin_manifest(&manifest).expect("manifest should map"); for (proxy_seq, ready_at_ms) in [(0, 0), (1, 1)] { let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("segment fixture"); segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; } + for map in session.maps.values() { + gc.cache.write_bytes_and_commit(&map.cache_key, &[0; 10]).await.expect("map fixture"); + } session.maps.get_mut(&ProxyMapId(0)).expect("first map").status = MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; session.maps.get_mut(&ProxyMapId(1)).expect("second map").status = diff --git a/backend/hls/src/initial_strip.rs b/backend/hls/src/initial_strip.rs index 827d25938..ef17dccc5 100644 --- a/backend/hls/src/initial_strip.rs +++ b/backend/hls/src/initial_strip.rs @@ -268,3 +268,21 @@ mod tests { } } } + +#[cfg(test)] +mod short_window_tests { + use super::*; + + #[test] + fn strip_preserves_every_segment_of_an_already_short_playlist() { + for count in [1, 2] { + for mode in [HlsStripMode::Segments, HlsStripMode::Seconds] { + let strip = StripConfig { mode, value: 30 }; + let body = format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n{}", "#EXTINF:2.5,\nsegment.ts\n".repeat(count)); + let view = materialize_initial_hls_strip_view(&body, &strip, HlsManifestWindowPolicy::ApplyLiveWindow); + assert_eq!(view.body, body); + assert_eq!(initial_hls_strip_segments_for_durations(&strip, &vec![2_500; count]), 0); + } + } + } +} diff --git a/backend/hls/src/late_custom_response.rs b/backend/hls/src/late_custom_response.rs new file mode 100644 index 000000000..2233a80e9 --- /dev/null +++ b/backend/hls/src/late_custom_response.rs @@ -0,0 +1,168 @@ +//! Finite custom media published after the live splice deadline has elapsed. + +use crate::{ + hls_ctx::HlsCtx, + lease::{HlsAccessLeaseId, HlsTerminalTailPreparation}, + prepared_terminal_bundle::{prepared_terminal_bundle_key, HlsPreparedTerminalBundle}, + runtime_custom_tail::{ + await_prepared_standalone_bundle, snapshot_hls_runtime_custom_tail_asset, standalone_custom_asset_fingerprint, + HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, HlsStandaloneCustomPlan, + HlsStandaloneCustomPlanKey, + }, + terminal_tail::{HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailGeneration}, + HlsAccessLeaseState, ProxySessionId, +}; +use std::{fmt, fmt::Write, sync::Arc}; +use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs}; + +/// Immutable, lease-bound custom playlist that does not consume a live splice base. +#[derive(Clone)] +pub struct HlsLateCustomResponsePlan { + pub generation: HlsTerminalTailGeneration, + pub(crate) manifest_snapshot_generation: u64, + pub(crate) key: HlsStandaloneCustomPlanKey, + proxy_session_id: ProxySessionId, + lease_id: HlsAccessLeaseId, + standalone: HlsStandaloneCustomPlan, +} + +impl HlsLateCustomResponsePlan { + pub fn matches_route(&self, proxy_session_id: &ProxySessionId, lease_id: &HlsAccessLeaseId) -> bool { + self.proxy_session_id == *proxy_session_id && self.lease_id == *lease_id + } + + pub fn manifest_body(&self) -> &str { &self.standalone.manifest_body } + + pub(crate) fn segment_bytes( + &self, + fingerprint: &str, + index: u16, + ) -> Result { + use crate::runtime_custom_tail::HlsStandaloneCustomSegmentError; + if fingerprint != standalone_custom_asset_fingerprint(self.key.asset) { + return Err(HlsStandaloneCustomSegmentError::StaleAssetFingerprint); + } + self.standalone.segment_bytes(index).ok_or(HlsStandaloneCustomSegmentError::InvalidIndex) + } +} + +impl fmt::Debug for HlsLateCustomResponsePlan { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("HlsLateCustomResponsePlan") + .field("generation", &self.generation) + .field("key", &self.key) + .finish_non_exhaustive() + } +} + +impl PartialEq for HlsLateCustomResponsePlan { + fn eq(&self, other: &Self) -> bool { + self.generation == other.generation + && self.manifest_snapshot_generation == other.manifest_snapshot_generation + && self.key == other.key + && self.proxy_session_id == other.proxy_session_id + && self.lease_id == other.lease_id + && self.standalone.manifest_body == other.standalone.manifest_body + } +} + +impl Eq for HlsLateCustomResponsePlan {} + +pub(crate) enum HlsLateCustomResponseError { + Superseded, + Unavailable(HlsTerminalTailCompatibility), +} + +pub(crate) async fn prepare_late_custom_response( + ctx: &HlsCtx, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, +) -> Result<(Arc, HlsRuntimeCustomTailAssetIdentity), HlsLateCustomResponseError> { + let lease = ctx + .hls_proxy + .access_lease_response_snapshot(lease_id, proxy_session_id, now_ms) + .await + .ok_or(HlsLateCustomResponseError::Superseded)?; + if !matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) + || lease.playback_mode != HlsLeasePlaybackMode::Live + || lease.issued_at_ms != preparation.lease_issued_at_ms + || lease.admission_generation != preparation.expected_admission_generation + || lease.playback_cursor.cursor_generation != preparation.cursor_generation + || lease.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation) + != Some(preparation.manifest_snapshot_generation) + { + return Err(HlsLateCustomResponseError::Superseded); + } + let snapshot = snapshot_hls_runtime_custom_tail_asset(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable) + .map_err(|error| HlsLateCustomResponseError::Unavailable(error.terminal_compatibility()))?; + let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&snapshot); + let bundle_key = prepared_terminal_bundle_key( + &snapshot.asset, + preparation.manifest_snapshot.target_duration_ms, + crate::terminal_tail::HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let bundle = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), bundle_key) + .await + .map_err(HlsLateCustomResponseError::Unavailable)?; + let first_proxy_seq = lease + .published_media_sequence_highwater + .unwrap_or(preparation.manifest_snapshot.last_proxy_seq) + .checked_add(1) + .ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?; + let discontinuity_sequence = + lease.published_discontinuity_sequence_highwater.max(preparation.manifest_snapshot.discontinuity_sequence); + let path_prefix = preparation + .manifest_snapshot + .visible_segments + .first() + .and_then(|segment| { + let uri = preparation.manifest_snapshot.materialize_uri(&segment.uri); + uri.find("/hls/shared/live/").map(|offset| uri[..offset].to_owned()) + }) + .ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::InvalidLeaseRoute))?; + let manifest_body = render_late_manifest(first_proxy_seq, discontinuity_sequence, &path_prefix, lease_id, &bundle)?; + let key = HlsStandaloneCustomPlanKey { + reason: snapshot.reason, + asset: expected_asset.media, + segment_count: bundle.key.segment_count, + }; + let standalone = HlsStandaloneCustomPlan { key, manifest_body: Arc::from(manifest_body), prepared_bundle: bundle }; + Ok(( + Arc::new(HlsLateCustomResponsePlan { + generation: HlsTerminalTailGeneration(preparation.decision_generation), + manifest_snapshot_generation: preparation.manifest_snapshot_generation, + key, + proxy_session_id: proxy_session_id.clone(), + lease_id: lease_id.clone(), + standalone, + }), + expected_asset, + )) +} + +// Keeping the layout separate makes the publication boundary testable without a live origin or cache. +fn render_late_manifest( + first_proxy_seq: u64, + discontinuity_sequence: u64, + path_prefix: &str, + lease_id: &HlsAccessLeaseId, + bundle: &HlsPreparedTerminalBundle, +) -> Result { + first_proxy_seq + .checked_add(u64::from(bundle.key.segment_count).saturating_sub(1)) + .ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?; + let fingerprint = standalone_custom_asset_fingerprint(bundle.key.asset); + let extinf = format_hls_duration_ms(bundle.source_asset_duration_ms); + let mut manifest = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{}\n#EXT-X-MEDIA-SEQUENCE:{first_proxy_seq}\n#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuity_sequence}\n#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n", hls_target_duration_secs(bundle.key.target_duration_ms)); + for index in 0..bundle.key.segment_count { + let _ = writeln!(manifest, "#EXTINF:{extinf},\n{path_prefix}/cvs/hls/{}/{fingerprint}/{index}.ts", lease_id.0); + } + manifest.push_str("#EXT-X-ENDLIST\n"); + Ok(manifest) +} + +#[cfg(test)] +mod tests; diff --git a/backend/hls/src/late_custom_response/tests.rs b/backend/hls/src/late_custom_response/tests.rs new file mode 100644 index 000000000..12f0a490c --- /dev/null +++ b/backend/hls/src/late_custom_response/tests.rs @@ -0,0 +1,47 @@ +use super::*; +use crate::{ + prepared_terminal_bundle::{HlsPreparedTerminalBundleKey, HlsPreparedTerminalSegment}, + terminal_tail::HlsTerminalAssetIdentity, +}; + +fn bundle() -> HlsPreparedTerminalBundle { + HlsPreparedTerminalBundle { + key: HlsPreparedTerminalBundleKey { + asset: HlsTerminalAssetIdentity { revision: 7, fingerprint: [7; 32] }, + target_duration_ms: 12_000, + segment_count: 12, + }, + source_asset_duration_ms: 10_027, + source_asset_duration_ticks_90khz: 902_400, + segments: (0..12) + .map(|index| HlsPreparedTerminalSegment { + index, + timestamp_offset_ticks_90khz: u64::from(index) * 902_400, + bytes: bytes::Bytes::from_static(b"prepared terminal segment"), + }) + .collect(), + } +} + +#[test] +fn late_manifest_keeps_target_and_sequences_and_contains_only_custom_media() { + let bundle = bundle(); + let manifest = render_late_manifest(131, 9, "/iptv", &HlsAccessLeaseId("lease".into()), &bundle) + .unwrap_or_else(|_| panic!("valid late manifest")); + assert!(manifest.contains("#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:131\n#EXT-X-DISCONTINUITY-SEQUENCE:9\n")); + assert_eq!(manifest.matches("#EXTINF:10.027,").count(), 12); + assert_eq!(manifest.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + assert!(manifest.contains("#EXT-X-KEY:METHOD=NONE\n")); + assert!(manifest.contains("/iptv/cvs/hls/lease/0000000000000007/11.ts")); + assert!(!manifest.contains("/hls/shared/live/")); + assert!(manifest.ends_with("#EXT-X-ENDLIST\n")); +} + +#[test] +fn late_manifest_rejects_sequence_overflow_instead_of_reusing_live_sequences() { + let bundle = bundle(); + assert!(matches!( + render_late_manifest(u64::MAX - 1, 0, "", &HlsAccessLeaseId("lease".into()), &bundle,), + Err(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed)) + )); +} diff --git a/backend/hls/src/lease.rs b/backend/hls/src/lease.rs index f7659a6b7..46c859e18 100644 --- a/backend/hls/src/lease.rs +++ b/backend/hls/src/lease.rs @@ -144,6 +144,8 @@ pub struct HlsAccessLease { pub startup_admission: HlsLeaseStartupAdmissionState, pub playback_cursor: HlsLeasePlaybackCursor, pub last_manifest_snapshot: Option, + pub(crate) published_media_sequence_highwater: Option, + pub(crate) published_discontinuity_sequence_highwater: u64, revision_bindings: Option>, progressive_startup_claimed: bool, published_transient_resource_ids: HlsPublishedTransientResourceIds, diff --git a/backend/hls/src/lease/identity.rs b/backend/hls/src/lease/identity.rs index 6fcde43a7..5908eecec 100644 --- a/backend/hls/src/lease/identity.rs +++ b/backend/hls/src/lease/identity.rs @@ -37,7 +37,9 @@ impl HlsAccessLease { HlsLeasePlaybackMode::TerminalTail(plan) => { HlsMediaLeasePlaybackIdentity::TerminalTail { generation: plan.generation } } - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return None, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => return None, }; Some(HlsMediaLeaseIdentity { issued_at_ms: self.issued_at_ms, playback }) } diff --git a/backend/hls/src/lease/lifecycle.rs b/backend/hls/src/lease/lifecycle.rs index 696d78b6f..d04d5cf29 100644 --- a/backend/hls/src/lease/lifecycle.rs +++ b/backend/hls/src/lease/lifecycle.rs @@ -93,7 +93,9 @@ impl HlsAccessLease { HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => { Some(HlsTerminalTailGeneration(decision_generation)) } - HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, + HlsLeasePlaybackMode::Live + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::Ended => None, }; if self.pending_terminal_protection_release.is_none() { self.pending_terminal_protection_release = release_generation; @@ -370,6 +372,24 @@ impl HlsAccessLeaseStore { Some(token) } + pub(crate) fn record_timed_segment_request_started_if_identity_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsMediaLeaseIdentity, + segment: &crate::SegmentEntry, + requested_at_ms: u64, + ) -> Option { + let token = self.record_segment_request_started_if_identity_matches( + lease_id, + proxy_session_id, + expected, + segment.proxy_seq, + requested_at_ms, + )?; + Some(self.by_lease_id.get_mut(lease_id)?.playback_cursor.bind_request_timing(token, segment)) + } + pub fn record_segment_request_completed_if_identity_matches( &mut self, lease_id: &HlsAccessLeaseId, diff --git a/backend/hls/src/lease/publication.rs b/backend/hls/src/lease/publication.rs index d0a8ad68d..c3dd770a8 100644 --- a/backend/hls/src/lease/publication.rs +++ b/backend/hls/src/lease/publication.rs @@ -154,6 +154,17 @@ impl HlsAccessLeaseStore { } lease.manifest_snapshot_generation = next_generation; snapshot.snapshot_generation = next_generation; + lease.published_media_sequence_highwater = Some( + lease + .published_media_sequence_highwater + .map_or(snapshot.last_proxy_seq, |previous| previous.max(snapshot.last_proxy_seq)), + ); + let last_discontinuity_sequence = snapshot.discontinuity_sequence.saturating_add( + u64::try_from(snapshot.visible_segments.iter().filter(|segment| segment.discontinuity_before).count()) + .unwrap_or(u64::MAX), + ); + lease.published_discontinuity_sequence_highwater = + lease.published_discontinuity_sequence_highwater.max(last_discontinuity_sequence); let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation; lease.last_manifest_snapshot = Some(snapshot); lease.published_transient_resource_ids = published_transient_resource_ids; diff --git a/backend/hls/src/lease/revision_bindings.rs b/backend/hls/src/lease/revision_bindings.rs index 3bc1bcb99..a76c7e3ec 100644 --- a/backend/hls/src/lease/revision_bindings.rs +++ b/backend/hls/src/lease/revision_bindings.rs @@ -30,6 +30,13 @@ impl HlsLeaseRevisionBindings { impl HlsAccessLeaseStore { pub fn prune_revision_bindings(&mut self, session: &ProxySessionId, first_retained_seq: u64) { for lease in self.by_lease_id.values_mut().filter(|lease| lease.proxy_session_id == *session) { + if let Some(publication) = + lease.last_manifest_snapshot.as_mut().and_then(|snapshot| snapshot.startup_revisions.as_mut()) + { + if publication.revisions.first_key_value().is_some_and(|(seq, _)| *seq < first_retained_seq) { + Arc::make_mut(publication).revisions.retain(|seq, _| *seq >= first_retained_seq); + } + } let Some(bindings) = &mut lease.revision_bindings else { continue }; if bindings.revisions.first_key_value().is_some_and(|(seq, _)| *seq < first_retained_seq) { Arc::make_mut(bindings).retire_while(|seq, _| seq < first_retained_seq); diff --git a/backend/hls/src/lease/runtime_policy.rs b/backend/hls/src/lease/runtime_policy.rs index a3794c35d..487cd922c 100644 --- a/backend/hls/src/lease/runtime_policy.rs +++ b/backend/hls/src/lease/runtime_policy.rs @@ -16,7 +16,9 @@ impl HlsAccessLease { .runtime_policy_revocation .clone() .map(|token| HlsRuntimePolicyRevocationOutcome::AlreadyPending { token }), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => None, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => None, } } @@ -24,9 +26,9 @@ impl HlsAccessLease { match &self.playback_mode { HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), HlsLeasePlaybackMode::Live => self.runtime_policy_revocation.as_ref().map(|token| token.reason), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { - self.runtime_policy_denial_reason - } + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => self.runtime_policy_denial_reason, } } @@ -177,13 +179,16 @@ impl HlsAccessLeaseStore { } let preserve_finite_decision = matches!( lease.playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } + HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } ); if preserve_finite_decision { if evidence_changed { lease.runtime_policy_denial_reason = match &lease.playback_mode { HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), - HlsLeasePlaybackMode::TerminalUnavailable { .. } => lease.runtime_policy_denial_reason, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } => lease.runtime_policy_denial_reason, HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, }; } diff --git a/backend/hls/src/lease/terminal.rs b/backend/hls/src/lease/terminal.rs index 63a202782..06d43e642 100644 --- a/backend/hls/src/lease/terminal.rs +++ b/backend/hls/src/lease/terminal.rs @@ -85,9 +85,14 @@ fn terminal_commit_existing_outcome( { Some(HlsTerminalCommitOutcome::AlreadyCommitted) } - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { - Some(HlsTerminalCommitOutcome::SupersededGeneration) + HlsLeasePlaybackMode::TerminalCustomResponse(current) + if current.generation.0 == preparation.decision_generation => + { + Some(HlsTerminalCommitOutcome::AlreadyCommitted) } + HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } => Some(HlsTerminalCommitOutcome::SupersededGeneration), HlsLeasePlaybackMode::Ended => Some(HlsTerminalCommitOutcome::LeaseNoLongerEligible), } } @@ -156,6 +161,7 @@ impl HlsAccessLeaseStore { if !matches!( lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended ) { @@ -286,6 +292,109 @@ impl HlsAccessLeaseStore { HlsTerminalCommitOutcome::Committed } + pub(crate) fn commit_terminal_custom_response_if_generation_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, + plan: std::sync::Arc, + ) -> HlsTerminalCommitOutcome { + if !plan.matches_route(proxy_session_id, lease_id) + || plan.generation.0 != preparation.decision_generation + || plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation + || preparation.trigger.is_runtime_policy() + { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) { + return outcome; + } + if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms) + { + return outcome; + } + if self.availability_evidence_can_advance().is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + lease.admission_generation = preparation.decision_generation; + lease.playback_mode = HlsLeasePlaybackMode::TerminalCustomResponse(plan); + if self.advance_availability_evidence(proxy_session_id).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + HlsTerminalCommitOutcome::Committed + } + + pub(crate) fn late_custom_segment_access( + &mut self, + lease_id: &HlsAccessLeaseId, + fingerprint: &str, + index: u16, + now_ms: u64, + ) -> Option< + Result< + crate::runtime_custom_tail::HlsStandaloneCustomSegmentAccess, + crate::runtime_custom_tail::HlsStandaloneCustomSegmentError, + >, + > { + use crate::runtime_custom_tail::{HlsStandaloneCustomSegmentAccess, HlsStandaloneSharedLeaseBinding}; + self.refresh_access_lease_validity(lease_id, now_ms).ok()?; + let lease = self.by_lease_id.get(lease_id)?; + let HlsLeasePlaybackMode::TerminalCustomResponse(plan) = &lease.playback_mode else { + return None; + }; + if !lease_state_allows_use(lease.state) || now_ms >= lease.valid_until_ms { + return Some(Err(crate::runtime_custom_tail::HlsStandaloneCustomSegmentError::UnknownAccessLease)); + } + Some(plan.segment_bytes(fingerprint, index).map(|bytes| HlsStandaloneCustomSegmentAccess { + terminal_generation: Some(plan.generation), + username: std::sync::Arc::from(lease.username.as_str()), + shared_lease: Some(HlsStandaloneSharedLeaseBinding { + proxy_session_id: lease.proxy_session_id.clone(), + lease_issued_at_ms: lease.issued_at_ms, + }), + bytes, + })) + } + + pub(crate) fn activate_late_custom_segment_access( + &mut self, + lease_id: &HlsAccessLeaseId, + access: &crate::runtime_custom_tail::HlsStandaloneCustomSegmentAccess, + now_ms: u64, + timing: super::HlsAccessLeaseTiming, + ) -> Option { + let binding = access.shared_lease.as_ref()?; + let generation = access.terminal_generation?; + let lease = self.by_lease_id.get(lease_id)?; + let HlsLeasePlaybackMode::TerminalCustomResponse(plan) = &lease.playback_mode else { + return None; + }; + if lease.issued_at_ms != binding.lease_issued_at_ms + || lease.username != access.username.as_ref() + || plan.generation != generation + || !plan.matches_route(&binding.proxy_session_id, lease_id) + { + return None; + } + if let super::HlsAccessLeaseActivation::Activated { lease, .. } = + self.activate_access_lease(lease_id, &binding.proxy_session_id, now_ms, timing) + { + Some(*lease) + } else { + None + } + } + pub fn terminal_tail_replay_outcome( &mut self, lease_id: &HlsAccessLeaseId, diff --git a/backend/hls/src/lease/tests/behavior.rs b/backend/hls/src/lease/tests/behavior.rs index f66f78368..f783bc7ba 100644 --- a/backend/hls/src/lease/tests/behavior.rs +++ b/backend/hls/src/lease/tests/behavior.rs @@ -286,7 +286,7 @@ fn session_follow_up_shortens_pending_lease_once() { 3_000, HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }, ); - assert!(unchanged.is_empty()); + assert_eq!(unchanged.len(), 0); let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); assert_eq!(lease.valid_until_ms, 12_000); @@ -514,7 +514,7 @@ fn bulk_lease_removal_advances_and_preserves_session_evidence_generation() { assert_eq!(store.remove_access_leases_for_session(&proxy_session_id).len(), 2); let after_removal = store.availability_evidence_generation(&proxy_session_id); assert!(after_removal > before_removal); - assert!(store.remove_access_leases_for_session(&proxy_session_id).is_empty()); + assert_eq!(store.remove_access_leases_for_session(&proxy_session_id).len(), 0); assert_eq!(store.availability_evidence_generation(&proxy_session_id), after_removal); } diff --git a/backend/hls/src/lease/timing.rs b/backend/hls/src/lease/timing.rs index 217552dc5..0e726ef4b 100644 --- a/backend/hls/src/lease/timing.rs +++ b/backend/hls/src/lease/timing.rs @@ -56,6 +56,8 @@ impl HlsAccessLease { revision_bindings: None, progressive_startup_claimed: false, last_manifest_snapshot: None, + published_media_sequence_highwater: None, + published_discontinuity_sequence_highwater: 0, published_transient_resource_ids: HlsPublishedTransientResourceIds::default(), published_finalized_manifest_generations: BTreeSet::new(), manifest_snapshot_generation: 0, diff --git a/backend/hls/src/lib.rs b/backend/hls/src/lib.rs index fe45135b1..2f09be4dd 100644 --- a/backend/hls/src/lib.rs +++ b/backend/hls/src/lib.rs @@ -48,6 +48,7 @@ mod gc; pub mod header_policy; mod headers; mod ids; +mod late_custom_response; mod lease; mod lifecycle; mod manager; @@ -63,6 +64,8 @@ mod master_playlist; mod media_reserve; mod observability; mod origin; +mod origin_account_runtime; +mod origin_attempt; mod origin_progress; mod paths; mod post_refresh_availability; @@ -84,6 +87,7 @@ mod segment_revision; mod segment_watchdog; mod session; mod session_store; +mod short_manifest; mod startup_observability; mod startup_policy; mod sync_ext; @@ -128,6 +132,7 @@ pub mod api { scrub_hls_origin_headers, should_remove_hls_origin_header, }, ids::{build_proxy_session_id, HlsSessionKey, ProxySessionId, HLS_ACCESS_LEASE_ID_PLACEHOLDER}, + late_custom_response::HlsLateCustomResponsePlan, lease::{ new_hls_access_lease_id, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, HlsAccessLeaseTiming, HlsAccessLeaseTouch, @@ -146,6 +151,7 @@ pub mod api { derive_hls_lease_manifest_snapshot, HlsLeaseManifestSnapshotInput, HlsTransientManifestTemplate, }, map::{MapCacheStatus, MapEntry, OriginMapKey, ProxyMapId}, + map_fetcher::MapFetchContext, master_playlist::{ HlsBandwidthPersistenceOutcome, HlsMasterBandwidth, HlsMasterBandwidthSelection, HlsSingleVariantMasterPlaylist, @@ -163,6 +169,9 @@ pub mod api { HlsOriginAccountDetachedReason, HlsOriginAccountStatus, HlsOriginIoContext, HlsOriginSource, HlsOriginSourceKind, HlsOriginWorkClass, }, + origin_account_runtime::{ + HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount, + }, origin_progress::publication_late_after_ms, paths::{HlsMapFile, HlsSegmentFile, TransientResourceFile}, playback::{ diff --git a/backend/hls/src/manager.rs b/backend/hls/src/manager.rs index b57067902..a97661bee 100644 --- a/backend/hls/src/manager.rs +++ b/backend/hls/src/manager.rs @@ -121,6 +121,7 @@ pub struct HlsTerminalCommitRequest<'a> { /// Media and compatibility evidence required for one terminal publication kind. pub enum HlsTerminalCommitPayload { Tail { plan: Arc, media_guard: HlsTerminalCommitMediaGuard }, + CustomResponse(Arc), Unavailable(HlsTerminalTailCompatibility), UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility), } diff --git a/backend/hls/src/manager/activity.rs b/backend/hls/src/manager/activity.rs index b066156d7..a703033e4 100644 --- a/backend/hls/src/manager/activity.rs +++ b/backend/hls/src/manager/activity.rs @@ -31,11 +31,15 @@ impl HlsProxyManager { proxy_seq: u64, requested_at_ms: u64, ) -> Option { - self.access_leases.write().await.record_segment_request_started_if_identity_matches( + let session = self.sessions.get_by_proxy_session_id(proxy_session_id).await?; + let mut leases = self.access_leases.write().await; + let session = session.read().await; + let segment = session.segments.get(&proxy_seq)?; + leases.record_timed_segment_request_started_if_identity_matches( lease_id, proxy_session_id, lease_identity, - proxy_seq, + segment, requested_at_ms, ) } diff --git a/backend/hls/src/manager/cleanup.rs b/backend/hls/src/manager/cleanup.rs index 2a9817f10..f3f1700f0 100644 --- a/backend/hls/src/manager/cleanup.rs +++ b/backend/hls/src/manager/cleanup.rs @@ -2,7 +2,6 @@ use super::{ safe_proxy_session_id, HlsAccessLeaseSessionSnapshot, HlsExpiredSessionMarker, HlsProxyManager, HlsSessionHandle, ProxySessionId, }; -use crate::sync_ext::MutexExt; use log::debug; use std::{ io, @@ -181,33 +180,12 @@ impl HlsProxyManager { } let report = self.gc.run_once(now_ms).await?; self.cleanup_after_garbage_collection(&report).await; - if self.revision_store.has_created_revisions() { - for handle in self.sessions.list_sessions().await { - let snapshot = { - let mut session = handle.write().await; - let super::super::HlsSession { segments, startup, proxy_session_id, .. } = &mut *session; - startup.as_mut().map(|startup| { - startup.revisions.retain(|seq, _| segments.contains_key(seq)); - (proxy_session_id.clone(), segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq)) - }) - }; - if let Some((id, floor)) = snapshot { - self.access_leases.write().await.prune_revision_bindings(&id, floor); - } - } - } if self.revision_reconciliation_pending.load(Ordering::Acquire) { let deleted = self.segment_cache.delete_orphan_revision_files(std::time::SystemTime::now()).await?; if deleted < super::super::cache::MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN { self.revision_reconciliation_pending.store(false, Ordering::Release); } } - for revision in self.revision_store.retire_unpinned()? { - revision.file_pin.lock_unpoisoned().take(); - if self.segment_cache.delete_if_inactive(&revision.key).await.is_err() { - self.revision_store.requeue_retired(revision)?; - } - } if self.revision_store.has_created_revisions() { let bytes = self.segment_cache.revision_disk_bytes(); self.metrics.record_revision_disk_bytes(bytes); diff --git a/backend/hls/src/manager/config.rs b/backend/hls/src/manager/config.rs index b415011bb..c1e7e54a1 100644 --- a/backend/hls/src/manager/config.rs +++ b/backend/hls/src/manager/config.rs @@ -107,6 +107,9 @@ impl HlsProxyManager { )); segment_cache.install_capacity_reclaimer(&gc); gc.install_access_leases(&access_leases); + let revision_store = Arc::new(super::super::SegmentRevisionStore::default()); + gc.install_revision_store(&revision_store); + segment_cache.install_revision_metrics(&metrics); let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); Self { sessions, @@ -126,7 +129,7 @@ impl HlsProxyManager { Some(Arc::clone(&availability_reevaluations)), )), progressive_budget: super::super::ProgressiveBudgetManager::new(runtime_config.startup.clone()), - revision_store: Arc::new(super::super::SegmentRevisionStore::default()), + revision_store, revision_reconciliation_pending: AtomicBool::new(true), runtime_config: ArcSwap::from_pointee(runtime_config), transient_resources: Arc::new(TransientResourceStore::new()), diff --git a/backend/hls/src/manager/publication.rs b/backend/hls/src/manager/publication.rs index cb0253f1b..8f6ac90c9 100644 --- a/backend/hls/src/manager/publication.rs +++ b/backend/hls/src/manager/publication.rs @@ -34,6 +34,7 @@ impl HlsTerminalCommitPayload { pub(super) fn into_parts(self) -> (HlsTerminalLeaseDecision, Option) { match self { Self::Tail { plan, media_guard } => (HlsTerminalLeaseDecision::Tail(plan), Some(media_guard)), + Self::CustomResponse(plan) => (HlsTerminalLeaseDecision::CustomResponse(plan), None), Self::Unavailable(reason) => (HlsTerminalLeaseDecision::Unavailable(reason), None), Self::UnavailableAfterOwnerFailure(reason) => { (HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), None) @@ -155,6 +156,9 @@ impl HlsProxyManager { )); segment_cache.install_capacity_reclaimer(&gc); gc.install_access_leases(&access_leases); + let revision_store = Arc::new(super::super::SegmentRevisionStore::default()); + gc.install_revision_store(&revision_store); + segment_cache.install_revision_metrics(&metrics); let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); Self { sessions, @@ -174,7 +178,7 @@ impl HlsProxyManager { Some(Arc::clone(&availability_reevaluations)), )), progressive_budget: super::super::ProgressiveBudgetManager::new(runtime_config.startup.clone()), - revision_store: Arc::new(super::super::SegmentRevisionStore::default()), + revision_store, revision_reconciliation_pending: AtomicBool::new(true), runtime_config: ArcSwap::from_pointee(runtime_config), transient_resources: Arc::new(TransientResourceStore::new()), diff --git a/backend/hls/src/manager/terminal.rs b/backend/hls/src/manager/terminal.rs index fea5d57db..73187ec70 100644 --- a/backend/hls/src/manager/terminal.rs +++ b/backend/hls/src/manager/terminal.rs @@ -71,9 +71,26 @@ fn evaluate_terminal_commit_authorization( if !terminal_media_requirement_is_bound_to_preparation(preparation) { return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); } + if let HlsTerminalLeaseDecision::CustomResponse(plan) = decision { + if preparation.trigger.is_runtime_policy() + || !plan.matches_route(&session.proxy_session_id, lease_id) + || plan.generation.0 != preparation.decision_generation + || plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation + { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); + } + if !session.origin_control.path_is_degraded_at(now_ms) { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); + } + if now_ms < preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible); + } + return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }; + } let protection_capacity_exceeded = matches!(decision, HlsTerminalLeaseDecision::Tail(_)) && !session.can_install_terminal_tail_protection(lease_id); let (terminal, terminal_preparation) = match (decision, protection_capacity_exceeded) { + (HlsTerminalLeaseDecision::CustomResponse(_), _) => unreachable!("custom response authorized above"), (HlsTerminalLeaseDecision::Tail(plan), false) => { let prepared_key = plan.media_preparation_key(); if preparation.required_terminal_media_key != Some(prepared_key) @@ -99,13 +116,12 @@ fn evaluate_terminal_commit_authorization( ) => (HlsTerminalCutoverCapability::TailUnavailable(*reason), preparation.terminal_media_preparation), }; if matches!(decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) { - if !session.origin_control.path_condition.is_degraded() { + if !session.origin_control.path_is_degraded_at(now_ms) { return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); } - // This fallback publishes no media bytes. Submission arbitration and - // the terminal precondition retain the preparation's original - // exclusive deadline, so this authorization is reachable only while - // the autonomous owner still has time to commit safely. + // This fallback publishes no media bytes. Submission arbitration gives it + // a bounded CAS budget independent of the expired media-splice deadline. + // Session, recovery and lease generations still have to match. return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }; } if preparation.trigger.is_runtime_policy() { @@ -190,16 +206,61 @@ impl HlsProxyManager { self.standalone_custom_access.register(entry, now_ms); } - pub fn resolve_standalone_custom_segment( + pub async fn resolve_standalone_custom_segment( &self, lease_id: &HlsAccessLeaseId, asset_fingerprint: &str, index: u16, now_ms: u64, ) -> Result { + if let Some(access) = + self.access_leases.write().await.late_custom_segment_access(lease_id, asset_fingerprint, index, now_ms) + { + return access; + } self.standalone_custom_access.resolve(lease_id, asset_fingerprint, index, now_ms) } + /// Records a successful authorized custom-media request for its exact terminal lease generation. + pub async fn record_standalone_custom_segment_activity( + &self, + lease_id: &HlsAccessLeaseId, + access: &HlsStandaloneCustomSegmentAccess, + now_ms: u64, + ) -> super::HlsMediaActivityCommitOutcome { + use super::{HlsCriticalHandoffStateAccess, HlsMediaActivityCommitOutcome}; + if access.terminal_generation.is_none() { + return HlsMediaActivityCommitOutcome::Committed; + } + let Some(binding) = access.shared_lease.as_ref() else { + return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; + }; + let Some(session) = self.sessions.get_by_proxy_session_id(&binding.proxy_session_id).await else { + return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; + }; + let result = self + .with_critical_handoff_state(&session, |leases, session| { + let timing = super::HlsAccessLeaseTiming { + active_window_ms: session.account_overlap_timing().hard_active_window_ms.saturating_mul(2), + valid_window_ms: self.session_idle_timeout_ms(), + }; + let lease = leases.activate_late_custom_segment_access(lease_id, access, now_ms, timing)?; + session.mark_authorized_media_access(now_ms); + Some(lease) + }) + .await; + match result { + HlsCriticalHandoffStateAccess::Acquired(Some(lease)) => { + self.schedule_access_lease_activity(&lease).await; + self.schedule_access_lease_validity(&lease).await; + self.schedule_session_idle_for_handle(&session).await; + HlsMediaActivityCommitOutcome::Committed + } + HlsCriticalHandoffStateAccess::Acquired(None) => HlsMediaActivityCommitOutcome::StaleLeaseIdentity, + HlsCriticalHandoffStateAccess::LockBusy => HlsMediaActivityCommitOutcome::DeferredLockContention, + } + } + pub fn terminal_pending(&self) -> Arc { Arc::clone(&self.terminal_pending) } /// Cancels terminal work frozen before newly committed shared media @@ -277,7 +338,7 @@ impl HlsProxyManager { session.activity.media_readiness_generation, session.origin_control.origin_epoch, session.origin_control.last_media_progress_at_ms, - session.origin_control.path_condition.is_degraded(), + session.origin_control.path_is_degraded_at(now_ms), recovery_snapshot, ) }; @@ -470,7 +531,9 @@ impl HlsProxyManager { match command.asset_revision_guard.validate_current() { HlsTerminalAssetRevisionValidation::Current => {} HlsTerminalAssetRevisionValidation::Changed { current } => { - if command.preparation.trigger.is_runtime_policy() { + if command.preparation.trigger.is_runtime_policy() + || matches!(decision, HlsTerminalLeaseDecision::CustomResponse(_)) + { return HlsTerminalCommitAttempt::Completed(HlsTerminalCommitOutcome::SupersededGeneration); } let reason = current.asset.map_or(HlsTerminalTailCompatibility::MissingAsset, |_| { @@ -525,7 +588,7 @@ impl HlsProxyManager { let proxy_session_id = &command.key.proxy_session_id; let preparation = &command.preparation; let replay = match &command.decision { - HlsTerminalLeaseDecision::Tail(_) => { + HlsTerminalLeaseDecision::Tail(_) | HlsTerminalLeaseDecision::CustomResponse(_) => { leases.terminal_tail_replay_outcome(lease_id, proxy_session_id, preparation, now_ms) } HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) => { @@ -545,7 +608,8 @@ impl HlsProxyManager { && session.activity.media_readiness_generation == preparation.media_readiness_generation && session.origin_control.origin_epoch == preparation.origin_epoch && session.origin_control.last_media_progress_at_ms == preparation.last_media_progress_at_ms - && hls_key_readiness_evidence_is_current(preparation.reserve.key_readiness_valid_until_ms, now_ms) + && (matches!(command.decision, HlsTerminalLeaseDecision::CustomResponse(_)) + || hls_key_readiness_evidence_is_current(preparation.reserve.key_readiness_valid_until_ms, now_ms)) { return None; } @@ -575,7 +639,9 @@ impl HlsProxyManager { }), (HlsTerminalLeaseDecision::Tail(_), true) | ( - HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_), + HlsTerminalLeaseDecision::CustomResponse(_) + | HlsTerminalLeaseDecision::Unavailable(_) + | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_), _, ) => None, }; @@ -595,6 +661,14 @@ impl HlsProxyManager { } } let outcome = match (decision, protection_capacity_exceeded) { + (HlsTerminalLeaseDecision::CustomResponse(plan), _) => leases + .commit_terminal_custom_response_if_generation_matches( + lease_id, + proxy_session_id, + preparation, + now_ms, + Arc::clone(plan), + ), (HlsTerminalLeaseDecision::Tail(plan), false) => leases.commit_terminal_tail_if_generation_matches( lease_id, proxy_session_id, diff --git a/backend/hls/src/manager/tests/support.rs b/backend/hls/src/manager/tests/support.rs index 2b082da8a..0e539b85d 100644 --- a/backend/hls/src/manager/tests/support.rs +++ b/backend/hls/src/manager/tests/support.rs @@ -252,6 +252,20 @@ pub(in crate::manager::tests) async fn live_media_fixture( let manager = HlsProxyManager::with_hls_cache_config(&config); let (session, _) = manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + { + use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + let OriginManifestParseOutcome::Normal(manifest) = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4,\n40.ts\n", + "http://origin.example/index.m3u8", + ) else { + panic!("live media fixture parses"); + }; + let mut session = session.write().await; + session.proxy_next_seq = Some(40); + session.apply_origin_manifest(&manifest).expect("live segment timeline"); + session.segments.get_mut(&40).expect("live segment").status = + crate::SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 }; + } let proxy_session_id = session.read().await.proxy_session_id.clone(); let lease_id = HlsAccessLeaseId(lease_name.to_string()); let lease = access_lease(lease_name, &proxy_session_id); diff --git a/backend/hls/src/manager/tests/terminal.rs b/backend/hls/src/manager/tests/terminal.rs index bb3f74d56..d8f911359 100644 --- a/backend/hls/src/manager/tests/terminal.rs +++ b/backend/hls/src/manager/tests/terminal.rs @@ -1253,3 +1253,32 @@ async fn hls_terminal_commit_protection_capacity_has_typed_unavailable_outcome() assert_eq!(session.terminal_tail_protection_count(), HLS_TERMINAL_TAIL_PROTECTION_CAPACITY); assert!(session.terminal_tail_protection(&lease_id).is_none()); } + +#[tokio::test] +async fn unavailable_after_expired_splice_deadline_still_rejects_recovered_origin_generation() { + let (manager, session, proxy_session_id, lease_id, preparation, _) = + prepared_terminal_commit_fixture("recovered-before-unavailable").await; + let now_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + 1; + manager.terminal_commit_clock.set_fixed_now_ms(now_ms); + session.write().await.origin_control.record_media_progress(now_ms, 12_000); + let outcome = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &preparation, + now_ms, + payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure( + HlsTerminalTailCompatibility::TerminalMediaNotReady, + ), + asset_revision_guard: HlsTerminalAssetRevisionGuard::matching_for_test(None), + }); + assert_eq!(outcome, HlsTerminalCommitOutcome::RecoveryCommitted); + assert_eq!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) + .await + .expect("recovered lease") + .playback_mode, + HlsLeasePlaybackMode::Live + ); +} diff --git a/backend/hls/src/manifest_acceptance/cohorts.rs b/backend/hls/src/manifest_acceptance/cohorts.rs index 92677cf33..136bf245e 100644 --- a/backend/hls/src/manifest_acceptance/cohorts.rs +++ b/backend/hls/src/manifest_acceptance/cohorts.rs @@ -1,4 +1,5 @@ use super::{ + evidence::HlsManifestWindowClass, timeline::{host_local_windows_compatible, same_host_timeline_compatible, strong_anchor_overlap}, HlsAlternativeOriginCohort, HlsAlternativeOriginCohortIdentity, HlsAlternativeOriginWindow, HlsCandidateHostRelation, HlsCrossHostAcceptanceEvidence, HlsHostLocalSequenceRelation, @@ -18,6 +19,7 @@ pub fn alternative_cohorts(observations: &[HlsManifestCandidateObservation]) -> .iter() .filter(|candidate| { candidate.host_relation == HlsCandidateHostRelation::OtherHost + && candidate.timeline_fingerprint.window_class() != HlsManifestWindowClass::Empty && candidate.resource_timeline_evidence.permits_acceptance() && candidate.switch_segment_readiness.can_be_staged() && candidate.effective_host.is_some() @@ -45,7 +47,12 @@ pub fn alternative_cohorts(observations: &[HlsManifestCandidateObservation]) -> let mut cohorts = groups.into_iter().filter_map(|samples| build_alternative_cohort(&samples, &pinned)).collect::>(); cohorts.sort_by_key(|cohort| { - (evidence_priority(cohort.evidence), std::cmp::Reverse(cohort.successful_samples), cohort.best_candidate_index) + ( + cohort.window.fingerprint.window_class(), + evidence_priority(cohort.evidence), + std::cmp::Reverse(cohort.successful_samples), + cohort.best_candidate_index, + ) }); cohorts } @@ -215,7 +222,9 @@ fn build_alternative_cohort( } else { HlsCrossHostAcceptanceEvidence::Insufficient }; - let best = samples.iter().min_by_key(|sample| (sample.manifest_fetch_elapsed_ms, sample.candidate_index))?; + let best = samples.iter().min_by_key(|sample| { + (sample.timeline_fingerprint.window_class(), sample.manifest_fetch_elapsed_ms, sample.candidate_index) + })?; let best_candidate_index = best.candidate_index; let effective_host = first.effective_host.clone()?; Some(HlsAlternativeOriginCohort { diff --git a/backend/hls/src/manifest_acceptance/episode.rs b/backend/hls/src/manifest_acceptance/episode.rs index 8e45c0500..00909bc05 100644 --- a/backend/hls/src/manifest_acceptance/episode.rs +++ b/backend/hls/src/manifest_acceptance/episode.rs @@ -218,6 +218,39 @@ impl HlsManifestAcceptanceEpisode { HlsRecoveryWorkloadBindingUpdate::Applied } + /// Reopens only the selection step of an authorized reduced retry. Holding owns + /// the selection reset; all workload and timing guards still run in `select_candidate`. + pub(crate) fn select_candidate_for_reduced_retry( + &mut self, + expected_generation: HlsManifestAcceptanceGeneration, + candidate_identity: HlsManifestRecoveryCandidateIdentity, + ) -> HlsRecoveryWorkloadBindingUpdate { + if self.generation != expected_generation { + return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; + } + if !self.full_burst_completed + || self.deterministic_conflict_receipt.is_some() + || !matches!( + self.outcome, + HlsManifestAcceptanceEpisodeOutcome::Pending + | HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted( + HlsManifestAcceptanceExhaustionReason::AllFailed + | HlsManifestAcceptanceExhaustionReason::NoProgress + | HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate + ) + ) + { + return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; + } + let previous_outcome = self.outcome; + self.outcome = HlsManifestAcceptanceEpisodeOutcome::Pending; + let update = self.select_candidate(expected_generation, candidate_identity); + if update != HlsRecoveryWorkloadBindingUpdate::Applied { + self.outcome = previous_outcome; + } + update + } + pub fn bind_selected_candidate( &mut self, expected_generation: HlsManifestAcceptanceGeneration, diff --git a/backend/hls/src/manifest_acceptance/evaluation.rs b/backend/hls/src/manifest_acceptance/evaluation.rs index 2ff6244bf..710f32e36 100644 --- a/backend/hls/src/manifest_acceptance/evaluation.rs +++ b/backend/hls/src/manifest_acceptance/evaluation.rs @@ -1,22 +1,16 @@ use super::{ - alternative_cohorts_with_history, HlsAlternativeOriginCohort, HlsCandidateHostRelation, - HlsCommittedContentAnchorEvidence, HlsCrossHostAcceptanceEvidence, HlsHostLocalSequenceRelation, - HlsManifestAcceptanceInput, HlsManifestAcceptanceTrigger, HlsManifestCandidateObservation, HlsManifestCommitKind, - HlsManifestCommitPlan, HlsSwitchSegmentReadiness, + alternative_cohorts_with_history, evidence::HlsManifestWindowClass, HlsAlternativeOriginCohort, + HlsCandidateHostRelation, HlsCommittedContentAnchorEvidence, HlsCrossHostAcceptanceEvidence, + HlsHostLocalSequenceRelation, HlsManifestAcceptanceInput, HlsManifestAcceptanceTrigger, + HlsManifestCandidateObservation, HlsManifestCommitKind, HlsManifestCommitPlan, HlsSwitchSegmentReadiness, }; pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> HlsManifestCommitPlan { - let (progressed_pinned, unchanged_pinned) = best_pinned_candidates(input.observations); - if let Some(pinned) = progressed_pinned { - return pinned_commit(pinned); - } - - // An unchanged pinned response is useful while reserve remains. Once the - // recovery deadline is reached it must not hide a qualified alternative. - if !input.trigger.recovery_required() { - if let Some(pinned) = unchanged_pinned { - return pinned_commit(pinned); - } + let pinned = preferred_pinned_candidate(input); + if let Some(candidate) = + pinned.filter(|candidate| candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) + { + return pinned_commit(candidate); } if !input.full_burst_completed { @@ -27,32 +21,46 @@ pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> Hl }; } - if let Some(initial) = input - .observations - .iter() - .filter(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::InitialBaseline - && candidate.resource_timeline_evidence.permits_acceptance() - }) - .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) - { - return pinned_commit(initial); - } - let cohorts = alternative_cohorts_with_history( input.observations, input.previous_alternative, input.current_burst_is_full_plan, ); + // Exhaust all existing evidence rules for regular windows before considering + // short fallbacks. Short observations remain available as timeline anchors. + for window_class in [HlsManifestWindowClass::Regular, HlsManifestWindowClass::Short] { + if let Some(initial) = initial_baseline_candidate(input.observations, window_class) { + return pinned_commit(initial); + } + if let Some(candidate) = pinned.filter(|_| window_class == HlsManifestWindowClass::Short) { + return pinned_commit(candidate); + } + if let Some(plan) = qualified_alternative_plan(input, &cohorts, window_class) { + return plan; + } + } + if cohorts.is_empty() { + HlsManifestCommitPlan::RejectAll + } else { + HlsManifestCommitPlan::HoldAlternative + } +} + +fn qualified_alternative_plan( + input: HlsManifestAcceptanceInput<'_>, + cohorts: &[HlsAlternativeOriginCohort], + window_class: HlsManifestWindowClass, +) -> Option { if let Some(cohort) = cohorts .iter() + .filter(|cohort| cohort.window.fingerprint.window_class() == window_class) .find(|cohort| matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { .. })) { - return alternative_plan( + return Some(alternative_plan( input.observations, cohort.best_candidate_index, HlsManifestCommitKind::AnchoredAlternative, - ); + )); } if let Some(candidate) = input @@ -62,7 +70,9 @@ pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> Hl .observations .iter() .filter(|candidate| candidate.host_relation == HlsCandidateHostRelation::OtherHost) + .filter(|candidate| candidate.timeline_fingerprint.window_class() == window_class) .filter(|candidate| candidate.resource_timeline_evidence.permits_acceptance()) + .filter(|candidate| candidate.switch_segment_readiness.can_be_staged()) .filter(|candidate| { candidate.committed_content_anchor == HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification @@ -71,41 +81,41 @@ pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> Hl }) .flatten() { - return alternative_plan( + return Some(alternative_plan( input.observations, candidate.candidate_index, HlsManifestCommitKind::ContentVerifiedAlternative, - ); + )); } if input.trigger.recovery_required() { - if let Some(cohort) = cohorts.iter().find(|cohort| { - matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. }) - || cohort.consecutive_confirmed_full_bursts >= 2 - }) { - return alternative_plan( + if let Some(cohort) = + cohorts.iter().filter(|cohort| cohort.window.fingerprint.window_class() == window_class).find(|cohort| { + matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. }) + || cohort.consecutive_confirmed_full_bursts >= 2 + }) + { + return Some(alternative_plan( input.observations, cohort.best_candidate_index, HlsManifestCommitKind::AlternativeAsNewEpoch, - ); + )); } } if input.trigger == HlsManifestAcceptanceTrigger::Critical { - if let Some(cohort) = critical_single_candidate_cohort(&cohorts, input.observations) { - return alternative_plan( + if let Some(cohort) = critical_single_candidate_cohort(cohorts, input.observations) + .filter(|cohort| cohort.window.fingerprint.window_class() == window_class) + { + return Some(alternative_plan( input.observations, cohort.best_candidate_index, HlsManifestCommitKind::EmergencyAlternativeAsNewEpoch, - ); + )); } } - if cohorts.is_empty() { - HlsManifestCommitPlan::RejectAll - } else { - HlsManifestCommitPlan::HoldAlternative - } + None } fn critical_single_candidate_cohort<'a>( @@ -150,12 +160,16 @@ fn alternative_plan( fn best_pinned_candidates( observations: &[HlsManifestCandidateObservation], + full_burst_completed: bool, ) -> (Option<&HlsManifestCandidateObservation>, Option<&HlsManifestCandidateObservation>) { let mut progressed = None; let mut unchanged = None; for candidate in observations.iter().filter(|candidate| { candidate.host_relation == HlsCandidateHostRelation::PinnedHost && candidate.resource_timeline_evidence.permits_acceptance() + && candidate.timeline_fingerprint.window_class() != HlsManifestWindowClass::Empty + && (full_burst_completed + || candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) }) { match candidate.local_sequence_relation { Some( @@ -180,8 +194,12 @@ fn best_pinned_candidates( } fn pinned_order(left: &HlsManifestCandidateObservation, right: &HlsManifestCandidateObservation) -> std::cmp::Ordering { - pinned_priority(left.local_sequence_relation) - .cmp(&pinned_priority(right.local_sequence_relation)) + left.timeline_fingerprint + .window_class() + .cmp(&right.timeline_fingerprint.window_class()) + .then_with(|| { + pinned_priority(left.local_sequence_relation).cmp(&pinned_priority(right.local_sequence_relation)) + }) .then_with(|| left.manifest_fetch_elapsed_ms.cmp(&right.manifest_fetch_elapsed_ms)) .then_with(|| left.candidate_index.cmp(&right.candidate_index)) } @@ -203,3 +221,30 @@ fn has_stageable_alternative(observations: &[HlsManifestCandidateObservation]) - && candidate.switch_segment_readiness.can_be_staged() }) } + +fn initial_baseline_candidate( + observations: &[HlsManifestCandidateObservation], + window_class: HlsManifestWindowClass, +) -> Option<&HlsManifestCandidateObservation> { + observations + .iter() + .filter(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::InitialBaseline + && candidate.resource_timeline_evidence.permits_acceptance() + && candidate.timeline_fingerprint.window_class() == window_class + }) + .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) +} + +fn preferred_pinned_candidate(input: HlsManifestAcceptanceInput<'_>) -> Option<&HlsManifestCandidateObservation> { + let (progressed_pinned, unchanged_pinned) = best_pinned_candidates(input.observations, input.full_burst_completed); + let eligible_unchanged = (!input.trigger.recovery_required()).then_some(unchanged_pinned).flatten(); + progressed_pinned + .filter(|candidate| candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) + .or_else(|| { + eligible_unchanged + .filter(|candidate| candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) + }) + .or(progressed_pinned) + .or(eligible_unchanged) +} diff --git a/backend/hls/src/manifest_acceptance/evidence.rs b/backend/hls/src/manifest_acceptance/evidence.rs index 7ff92624b..f36143bc8 100644 --- a/backend/hls/src/manifest_acceptance/evidence.rs +++ b/backend/hls/src/manifest_acceptance/evidence.rs @@ -41,6 +41,24 @@ pub struct HlsManifestTimelineFingerprint { pub segment_samples: Vec, } +/// Acceptance preference shared by cohort ranking and candidate selection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum HlsManifestWindowClass { + Regular, + Short, + Empty, +} + +impl HlsManifestTimelineFingerprint { + pub(super) const fn window_class(&self) -> HlsManifestWindowClass { + match self.segment_count { + 0 => HlsManifestWindowClass::Empty, + 1 | 2 => HlsManifestWindowClass::Short, + 3.. => HlsManifestWindowClass::Regular, + } + } +} + /// Stable, window-independent technical identity of an alternative timeline. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct HlsManifestTechnicalSignature { diff --git a/backend/hls/src/manifest_acceptance/tests.rs b/backend/hls/src/manifest_acceptance/tests.rs index 582606885..c35077926 100644 --- a/backend/hls/src/manifest_acceptance/tests.rs +++ b/backend/hls/src/manifest_acceptance/tests.rs @@ -106,3 +106,4 @@ fn sliding_observation( mod episode; mod evaluation; +mod reduced_retry; diff --git a/backend/hls/src/manifest_acceptance/tests/evaluation.rs b/backend/hls/src/manifest_acceptance/tests/evaluation.rs index 05be5fb91..e26bbdc9d 100644 --- a/backend/hls/src/manifest_acceptance/tests/evaluation.rs +++ b/backend/hls/src/manifest_acceptance/tests/evaluation.rs @@ -1,5 +1,86 @@ use super::*; +#[test] +fn regular_consensus_precedes_short_timeline_anchor_without_preferred_pinned_candidate() { + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); + let mut short = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + short.timeline_fingerprint.segment_count = 2; + short.timeline_fingerprint.segment_samples.truncate(2); + for (index, segment) in pinned.timeline_fingerprint.segment_samples.iter_mut().enumerate() { + segment.program_date_time_ms = Some(1_700_000_000_000 + i64::try_from(index).unwrap() * 4_000); + } + for (index, segment) in short.timeline_fingerprint.segment_samples.iter_mut().enumerate() { + segment.program_date_time_ms = Some(1_700_000_000_500 + i64::try_from(index).unwrap() * 4_000); + } + let regular = [ + observation(2, "origin-c", HlsCandidateHostRelation::OtherHost), + observation(3, "origin-c", HlsCandidateHostRelation::OtherHost), + ]; + for trigger in [HlsManifestAcceptanceTrigger::RecoveryRequired, HlsManifestAcceptanceTrigger::Critical] { + assert_eq!( + evaluate(&[pinned.clone(), short.clone(), regular[0].clone(), regular[1].clone()], trigger), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 2, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch, + } + ); + } + assert_eq!( + evaluate(&[pinned.clone(), short.clone(), regular[0].clone()], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::AnchoredAlternative, + } + ); + assert_eq!( + evaluate(&[pinned, short, regular[0].clone(), regular[1].clone()], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::AnchoredAlternative, + } + ); +} + +#[test] +fn regular_content_verification_precedes_short_anchor_and_unstageable_regular_does_not_block_fallback() { + let mut short = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + short.timeline_fingerprint.segment_count = 2; + short.timeline_fingerprint.segment_samples.truncate(2); + short.evidence = HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments: 2 }; + let mut regular = observation(1, "origin-c", HlsCandidateHostRelation::OtherHost); + regular.committed_content_anchor = HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification; + assert_eq!( + evaluate(&[short.clone(), regular.clone()], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::ContentVerifiedAlternative, + } + ); + regular.switch_segment_readiness = HlsSwitchSegmentReadiness::Unavailable; + assert_eq!( + evaluate(&[short, regular], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::AnchoredAlternative, + } + ); +} + +#[test] +fn empty_alternative_cannot_be_qualified_by_anchor_or_consensus() { + let mut empty = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + empty.timeline_fingerprint.segment_count = 0; + empty.timeline_fingerprint.segment_samples.clear(); + empty.evidence = HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments: 2 }; + let mut other = empty.clone(); + other.candidate_index = 1; + assert_eq!( + evaluate(&[empty, other], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::RejectAll + ); +} + #[test] fn forward_media_sequence_does_not_override_published_resource_replay() { let mut replay = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); @@ -342,7 +423,7 @@ fn conflicting_reduced_follow_up_cannot_reuse_previous_full_burst() { let mut conflict = observation(0, "origin-c", HlsCandidateHostRelation::OtherHost); conflict.timeline_fingerprint = fingerprint(20); - assert!(alternative_cohorts_with_history(&[conflict], Some(&first), false).is_empty()); + assert_eq!(alternative_cohorts_with_history(&[conflict], Some(&first), false).len(), 0); } #[test] @@ -399,3 +480,83 @@ fn critical_mode_rejects_ambiguous_single_sample_cohorts() { HlsManifestCommitPlan::HoldAlternative ); } + +#[test] +fn short_initial_and_pinned_candidates_require_a_complete_burst() { + for relation in [HlsCandidateHostRelation::InitialBaseline, HlsCandidateHostRelation::PinnedHost] { + for count in [1, 2] { + let mut candidate = observation(0, "origin-a", relation); + candidate.timeline_fingerprint.segment_count = count; + candidate.timeline_fingerprint.segment_samples.truncate(usize::try_from(count).unwrap()); + assert_eq!( + evaluate_manifest_acceptance(HlsManifestAcceptanceInput { + full_burst_completed: false, + current_burst_is_full_plan: true, + trigger: HlsManifestAcceptanceTrigger::Observe, + previous_alternative: None, + observations: std::slice::from_ref(&candidate), + }), + HlsManifestCommitPlan::RejectAll + ); + assert_eq!( + evaluate(&[candidate], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } + ); + } + } +} + +#[test] +fn eligible_regular_window_wins_over_faster_short_candidate_in_same_origin_class() { + for relation in [HlsCandidateHostRelation::InitialBaseline, HlsCandidateHostRelation::PinnedHost] { + let mut short = observation(0, "origin-a", relation); + short.timeline_fingerprint.segment_count = 1; + short.manifest_fetch_elapsed_ms = 1; + let mut regular = observation(1, "origin-a", relation); + regular.manifest_fetch_elapsed_ms = 100; + assert_eq!( + evaluate(&[short.clone(), regular.clone()], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::Commit { candidate_index: 1, kind: HlsManifestCommitKind::Pinned } + ); + regular.resource_timeline_evidence = HlsResourceTimelineEvidence::ReplayOnly; + assert_eq!( + evaluate(&[short, regular], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } + ); + } +} + +#[test] +fn empty_initial_candidates_are_not_qualified_by_a_complete_burst() { + let mut empty = observation(0, "origin-a", HlsCandidateHostRelation::InitialBaseline); + empty.timeline_fingerprint.segment_count = 0; + empty.timeline_fingerprint.segment_samples.clear(); + assert_eq!(evaluate(&[empty], HlsManifestAcceptanceTrigger::Observe), HlsManifestCommitPlan::RejectAll); +} + +#[test] +fn independently_qualified_regular_alternative_takes_priority_over_short_pinned_progress() { + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.timeline_fingerprint.segment_count = 1; + pinned.timeline_fingerprint.segment_samples.truncate(1); + let alternatives = [ + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(2, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + assert_eq!( + evaluate( + &[pinned.clone(), alternatives[0].clone(), alternatives[1].clone()], + HlsManifestAcceptanceTrigger::RecoveryRequired + ), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch + } + ); + let mut unstageable = alternatives[0].clone(); + unstageable.switch_segment_readiness = HlsSwitchSegmentReadiness::Unavailable; + assert_eq!( + evaluate(&[pinned, unstageable], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } + ); +} diff --git a/backend/hls/src/manifest_acceptance/tests/reduced_retry.rs b/backend/hls/src/manifest_acceptance/tests/reduced_retry.rs new file mode 100644 index 000000000..337c88712 --- /dev/null +++ b/backend/hls/src/manifest_acceptance/tests/reduced_retry.rs @@ -0,0 +1,190 @@ +use super::*; +use crate::manifest_acceptance::episode::HlsManifestAcceptanceEpisodeOutcome; +use shared::model::HlsManifestRecoveryBurstLevel; + +fn selectable_episode() -> HlsManifestAcceptanceEpisode { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let mut episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(7), + 100, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &episode_timing(100, plan), + ); + episode.record_full_burst(); + episode.state = HlsManifestAcceptanceState::Committing; + episode.observed_landscape = + Some(manifest_acceptance_landscape(&[observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost)])); + episode +} + +fn identity(marker: usize) -> HlsManifestRecoveryCandidateIdentity { + HlsManifestRecoveryCandidateIdentity::from_candidate(marker, Some("origin-a"), "manifest") +} + +fn assert_rejected_unchanged( + episode: &mut HlsManifestAcceptanceEpisode, + generation: HlsManifestAcceptanceGeneration, + candidate: HlsManifestRecoveryCandidateIdentity, + expected: HlsRecoveryWorkloadBindingUpdate, +) { + let before = episode.clone(); + assert_eq!(episode.select_candidate_for_reduced_retry(generation, candidate), expected); + assert_eq!(*episode, before, "a rejected selection is atomic"); +} + +#[test] +fn reduced_retry_reopens_only_eligible_exhaustion_and_preserves_frozen_evidence() { + for outcome in [ + HlsManifestAcceptanceEpisodeOutcome::Pending, + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(HlsManifestAcceptanceExhaustionReason::AllFailed), + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(HlsManifestAcceptanceExhaustionReason::NoProgress), + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted( + HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, + ), + ] { + let mut episode = selectable_episode(); + episode.outcome = outcome; + let generation = episode.generation; + let before = episode.clone(); + if outcome != HlsManifestAcceptanceEpisodeOutcome::Pending { + assert_eq!( + episode.select_candidate(generation, identity(0)), + HlsRecoveryWorkloadBindingUpdate::EpisodeInactive + ); + assert_eq!(episode, before, "normal selection still rejects exhaustion"); + } + let mut expected = before; + expected.outcome = HlsManifestAcceptanceEpisodeOutcome::Pending; + assert_eq!(expected.select_candidate(generation, identity(0)), HlsRecoveryWorkloadBindingUpdate::Applied); + assert_eq!( + episode.select_candidate_for_reduced_retry(generation, identity(0)), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + assert_eq!(episode, expected, "only outcome and the selected identity may change"); + } +} + +#[test] +fn reduced_retry_rejects_stale_generation_and_inactive_state_without_mutation() { + let mut episode = selectable_episode(); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoProgress); + let generation = episode.generation; + assert_rejected_unchanged( + &mut episode, + HlsManifestAcceptanceGeneration(generation.0 + 1), + identity(0), + HlsRecoveryWorkloadBindingUpdate::StaleGeneration, + ); + for state in [ + HlsManifestAcceptanceState::Holding, + HlsManifestAcceptanceState::Collecting, + HlsManifestAcceptanceState::Completed, + ] { + episode.state = state; + assert_rejected_unchanged( + &mut episode, + generation, + identity(0), + HlsRecoveryWorkloadBindingUpdate::EpisodeInactive, + ); + } +} + +#[test] +fn reduced_retry_keeps_identity_and_bound_workload_guards_until_holding_reset() { + let mut episode = selectable_episode(); + let generation = episode.generation; + assert_eq!(episode.select_candidate(generation, identity(0)), HlsRecoveryWorkloadBindingUpdate::Applied); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate); + assert_rejected_unchanged( + &mut episode, + generation, + identity(1), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch, + ); + + assert_eq!( + episode.select_candidate_for_reduced_retry(generation, identity(0)), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + let workload = HlsRecoveryWorkload { + burst: HlsRecoveryBurstWorkload::FullBurstCompleted, + segment: HlsRecoverySegmentWorkload::ClearSegmentFetch, + map: HlsRecoveryMapWorkload::NotRequired, + }; + assert_eq!( + episode.bind_selected_candidate(generation, identity(0), workload), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate); + assert_rejected_unchanged( + &mut episode, + generation, + identity(0), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch, + ); + assert_rejected_unchanged( + &mut episode, + generation, + identity(1), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch, + ); + + episode.hold_after_uncommitted_burst(None, Some(500)); + episode.state = HlsManifestAcceptanceState::Committing; + assert_eq!( + episode.select_candidate_for_reduced_retry(generation, identity(1)), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + assert_eq!(episode.selected_candidate_identity(), Some(identity(1))); +} + +#[test] +fn reduced_retry_never_reopens_incomplete_committed_or_deterministic_episodes() { + let mut episode = selectable_episode(); + let generation = episode.generation; + episode.full_burst_completed = false; + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + episode.full_burst_completed = true; + episode.complete(); + // Even an otherwise selectable state cannot reopen a committed outcome. + episode.state = HlsManifestAcceptanceState::Committing; + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + + let resource_key = HlsMediaResourceIdentity::for_test(7).semantic_key(); + let conflict = HlsDeterministicTimelineConflict { + previous_proxy_tail: Some(2), + existing_proxy_seq: 0, + candidate_position: 1, + candidate_origin_seq: 491, + resource_key, + decision: crate::timeline::HlsResourceReplayDecision::RejectContradictoryOrder, + candidate_fingerprint: crate::deterministic_conflict::HlsDeterministicConflictFingerprint { + segment_count: 1, + first_program_date_time_ms: None, + last_program_date_time_ms: None, + duration_pattern_hash: [1; 32], + discontinuity_pattern_hash: [2; 32], + semantic_resource_pattern_hash: Some(resource_key.bytes()), + map_and_encryption_hash: [3; 32], + container_signature_hash: [4; 32], + segment_samples: Vec::new(), + }, + }; + episode = selectable_episode(); + episode.record_deterministic_conflict(HlsDeterministicConflictReceipt { + conflict, + origin_progress_generation: 1, + published_resource_history_generation: 1, + pinned_host_generation: 1, + }); + episode.state = HlsManifestAcceptanceState::Committing; + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + // A receipt remains authoritative independently of how another caller recorded exhaustion. + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoProgress); + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + episode.deterministic_conflict_receipt = None; + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::DeterministicTimelineConflict); + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); +} diff --git a/backend/hls/src/manifest_commit.rs b/backend/hls/src/manifest_commit.rs index c454e4dd2..efa0c8d7f 100644 --- a/backend/hls/src/manifest_commit.rs +++ b/backend/hls/src/manifest_commit.rs @@ -183,6 +183,10 @@ fn origin_window_is_fresh(session: &HlsSession, now_ms: u64) -> bool { .is_some_and(|confirmed_at_ms| now_ms.saturating_sub(confirmed_at_ms) <= max_age_ms) } +pub(crate) fn has_servable_committed_manifest(session: &HlsSession, now_ms: u64) -> bool { + hls_committed_manifest_for_request(session, now_ms).is_some() +} + fn hls_committed_manifest_for_request(session: &HlsSession, now_ms: u64) -> Option { let protection = session.account_binding_protection(now_ms); hls_committed_manifest_candidate(session).filter(|candidate| { diff --git a/backend/hls/src/manifest_fetch/episode.rs b/backend/hls/src/manifest_fetch/episode.rs index 04c2dce07..41b6189a1 100644 --- a/backend/hls/src/manifest_fetch/episode.rs +++ b/backend/hls/src/manifest_fetch/episode.rs @@ -8,7 +8,10 @@ use super::{ current_time_millis, fingerprint::build_manifest_timeline_fingerprint, - recovery::{acceptance_attempt_may_start, HlsManifestRecoveryAttemptPlan, HlsManifestRecoveryCandidate}, + recovery::{ + acceptance_attempt_may_start, HlsManifestRecoveryAttemptPlan, HlsManifestRecoveryCandidate, + HlsManifestRequalificationWindow, + }, HlsManifestOriginRelation, HlsManifestSequenceRelation, HlsOriginManifestFetchContext, HLS_COMMITTED_CONTENT_ANCHOR_PROBE_LIMIT, }; @@ -23,32 +26,53 @@ use crate::{ HlsSwitchSegmentReadiness, }, recovery_timing::{ - HlsAcceptanceDeadlineMs, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, - HlsAcceptanceEpisodeTimingSeed, HlsRecoveryWorkloadEnvelope, HlsTerminalMediaPreparationState, - HlsTransitionMarginMs, + HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsAcceptanceEpisodeTimingSeed, + HlsRecoveryWorkloadEnvelope, HlsTerminalMediaPreparationState, HlsTransitionMarginMs, }, resource_identity::HlsMediaResourceIdentity, }; use log::debug; use shared::model::HlsManifestRecoveryBurstPlan; +pub(super) fn manifest_acceptance_generation_is_current( + session: &crate::HlsSession, + generation: HlsManifestAcceptanceGeneration, +) -> bool { + session.origin_control.acceptance_generation == generation + && session.origin_control.acceptance_episode.as_ref().is_some_and(|episode| episode.generation == generation) +} + pub(super) async fn begin_manifest_acceptance_episode( context: &HlsOriginManifestFetchContext, plan: &HlsManifestRecoveryAttemptPlan<'_>, burst_plan: HlsManifestRecoveryBurstPlan, + expected_generation: Option, ) -> Option { let mut session = context.session.write().await; + // Keep ownership validation atomic with the following episode/qualification changes. + if expected_generation.is_some_and(|generation| !manifest_acceptance_generation_is_current(&session, generation)) { + return None; + } + let basis = crate::short_manifest::HlsOriginQualificationBasis::capture(&session); + if !session.origin_control.qualification.pending_is_current(&basis) { + return None; + } if plan.attempt_index != 0 { let episode = session.origin_control.acceptance_episode.as_mut()?; if episode.trigger() != plan.trigger || episode.state == HlsManifestAcceptanceState::Completed { return None; } episode.state = HlsManifestAcceptanceState::Collecting; - return Some(episode.generation); + let generation = episode.generation; + session.origin_control.qualification = + crate::short_manifest::HlsOriginQualification::Checking { basis, generation, target: plan.binding.clone() }; + return Some(generation); } let started_at_ms = current_time_millis(); let timing = acceptance_episode_timing(context, &session, started_at_ms, burst_plan); let generation = session.origin_control.begin_acceptance_episode(started_at_ms, burst_plan, plan.trigger, &timing); + session.origin_control.qualification = + crate::short_manifest::HlsOriginQualification::Checking { basis, generation, target: plan.binding.clone() }; if let Some(episode) = session.origin_control.acceptance_episode.as_mut() { episode.state = HlsManifestAcceptanceState::Collecting; debug!( @@ -145,36 +169,51 @@ pub(super) async fn begin_requalified_manifest_acceptance_episode( context: &HlsOriginManifestFetchContext, generation: Option, trigger: HlsManifestAcceptanceTrigger, - acceptance_deadline: HlsAcceptanceDeadlineMs, -) -> bool { - let Some(generation) = generation else { - return false; - }; + window: &HlsManifestRequalificationWindow, +) -> Option { + let generation = generation?; let mut session = context.session.write().await; let now_ms = current_time_millis(); - if !acceptance_attempt_may_start(false, now_ms, 0, acceptance_deadline) { - return false; + match window { + HlsManifestRequalificationWindow::CurrentEpisode(deadline) + if !acceptance_attempt_may_start(false, now_ms, 0, *deadline) => + { + return None + } + HlsManifestRequalificationWindow::CurrentEpisode(_) + | HlsManifestRequalificationWindow::ResumedAfterSharedBackoff => {} } - let Some(episode) = session.origin_control.acceptance_episode.as_mut() else { - return false; - }; + let basis = crate::short_manifest::HlsOriginQualificationBasis::capture(&session); + if !matches!( + &session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Checking { basis: current, generation: current_generation, .. } + if current == &basis && *current_generation == generation + ) { + return None; + } + let episode = session.origin_control.acceptance_episode.as_mut()?; if episode.generation != generation || !episode.full_burst_completed || episode.state == HlsManifestAcceptanceState::Completed { - return false; + return None; } episode.state = HlsManifestAcceptanceState::Holding; let configured_plan = context.manifest_recovery_burst.level.plan(); let timing = acceptance_episode_timing(context, &session, now_ms, configured_plan); let next_generation = session.origin_control.begin_acceptance_episode(now_ms, configured_plan, trigger, &timing); + if let crate::short_manifest::HlsOriginQualification::Checking { generation, .. } = + &mut session.origin_control.qualification + { + *generation = next_generation; + } debug!( "HLS manifest acceptance landscape changed: previous_generation={} next_generation={} candidates={} decision=full-requalification", generation.0, next_generation.0, configured_plan.total_candidates() ); - true + Some(next_generation) } pub(super) async fn record_full_manifest_acceptance_burst( @@ -281,7 +320,10 @@ pub(super) async fn record_manifest_acceptance_exhaustion( let Some(episode) = session.origin_control.acceptance_episode.as_mut() else { return; }; - if episode.generation == generation { + if episode.generation == generation + && episode.exhaustion_reason().is_none() + && episode.deterministic_conflict_receipt().is_none() + { episode.record_exhaustion(reason); } } diff --git a/backend/hls/src/manifest_fetch/fetch.rs b/backend/hls/src/manifest_fetch/fetch.rs index 06c7c1640..a9e3e732e 100644 --- a/backend/hls/src/manifest_fetch/fetch.rs +++ b/backend/hls/src/manifest_fetch/fetch.rs @@ -5,15 +5,15 @@ use super::{ origin_manifest_fetch_error_from_io_error, resolved_hls_manifest_request_url_from_input, response_to_fetched_manifest, same_origin, ManifestRecoveryAttemptLogContext, }, - model::{FetchedOriginManifest, HlsOriginManifestFetchContext, RetryPolicy}, + model::{FetchedOriginManifest, HlsOriginManifestFetchContext, RetryPolicy, DEFAULT_HLS_TARGET_DURATION_SECS}, selection_log::{log_manifest_initial_attempt, log_manifest_retry_scheduled, ManifestRetryLogKind}, }; #[cfg(any(test, feature = "test-support"))] use super::{model::LiveHlsOriginEntry, selection_log::log_origin_refresh_retry_scheduled}; use crate::{ - append_hls_provider_session_headers, headers::merge_hls_provider_session_headers, - manifest_origin_binding::HlsManifestOriginBinding, observability::hls_origin_log_value, scrub_hls_origin_headers, - MAX_MANUAL_REDIRECTS, + append_hls_provider_session_headers, availability::record_retryable_manifest_attempt_failure, + headers::merge_hls_provider_session_headers, manifest_origin_binding::HlsManifestOriginBinding, + observability::hls_origin_log_value, scrub_hls_origin_headers, MAX_MANUAL_REDIRECTS, }; use axum::http::HeaderMap; use log::debug; @@ -82,13 +82,10 @@ async fn fetch_hls_origin_manifest_initial_global_policy( ) -> Result { log_manifest_initial_attempt(context).await; let input_source = context.origin_entry.to_input_source(); - let account_binding = { + let (account_binding, progress_generation) = { let session = context.session.read().await; - if session.origin_account_binding.is_some() { - "present" - } else { - "absent" - } + let account_binding = if session.origin_account_binding.is_some() { "present" } else { "absent" }; + (account_binding, session.origin_control.progress_generation) }; debug!( "HLS origin manifest request started: account_binding={account_binding} request_url={}", @@ -151,8 +148,13 @@ async fn fetch_hls_origin_manifest_initial_global_policy( match fetch_result { Ok(fetched) => return Ok(fetched.with_attempts(attempt_index + 1)), Err(err) if is_hls_retryable_initial_manifest_fetch_error(&err) && attempt_index + 1 < attempts => { + record_retryable_manifest_attempt_failure(context, progress_generation, &err, current_time_millis()) + .await; let retry_after_ms = match &err { - OriginManifestFetchError::RetryableStatus(_, retry_after_ms) => *retry_after_ms, + OriginManifestFetchError::RetryableStatus(_, Some(retry_after_ms)) => { + let session = context.session.read().await; + Some(cap_manifest_retry_after_ms(&session, *retry_after_ms)) + } _ => None, }; let jitter_ms = if retry_after_ms.is_some() { 0 } else { context.retry_policy.sample_jitter_ms() }; @@ -284,6 +286,16 @@ pub(super) fn single_variant_url(body: &str, master_url: &str) -> Option { matches!(url.scheme(), "http" | "https").then_some(url) } +pub(super) fn cap_manifest_retry_after_ms(session: &crate::HlsSession, retry_after_ms: u64) -> u64 { + let target_duration_ms = session + .origin_control + .target_duration_snapshot_ms + .filter(|duration_ms| *duration_ms > 0) + .or_else(|| session.target_duration.filter(|seconds| *seconds > 0).map(|seconds| u64::from(seconds) * 1_000)) + .unwrap_or(u64::from(DEFAULT_HLS_TARGET_DURATION_SECS) * 1_000); + retry_after_ms.min(target_duration_ms / 2) +} + pub(super) fn next_retry_delay_ms( retry_policy: &RetryPolicy, attempt_index: usize, diff --git a/backend/hls/src/manifest_fetch/model.rs b/backend/hls/src/manifest_fetch/model.rs index bd22705c0..e2e913d8c 100644 --- a/backend/hls/src/manifest_fetch/model.rs +++ b/backend/hls/src/manifest_fetch/model.rs @@ -240,6 +240,8 @@ impl FetchedOriginManifest { pub struct HlsOriginManifestFetchContext { pub app_config: Arc, pub session: HlsSessionHandle, + /// Availability owner used to react to failed attempts while the logical fetch is still in flight. + pub availability_runtime: Option, pub origin_entry: LiveHlsOriginEntry, pub headers: HeaderMap, pub client: Client, diff --git a/backend/hls/src/manifest_fetch/quality.rs b/backend/hls/src/manifest_fetch/quality.rs index 0c1974f0b..28422b32b 100644 --- a/backend/hls/src/manifest_fetch/quality.rs +++ b/backend/hls/src/manifest_fetch/quality.rs @@ -228,6 +228,9 @@ fn classify_manifest_sequence_relation( } fn manifest_continuity_mode(session: &crate::HlsSession, now_ms: u64) -> HlsManifestContinuityMode { + if session.origin_is_qualified() { + return HlsManifestContinuityMode::StrictContinuity; + } if session.origin_seq_highwater.is_none() { return HlsManifestContinuityMode::RebaseAllowed; } diff --git a/backend/hls/src/manifest_fetch/recovery.rs b/backend/hls/src/manifest_fetch/recovery.rs index bf37650b3..8f79e904c 100644 --- a/backend/hls/src/manifest_fetch/recovery.rs +++ b/backend/hls/src/manifest_fetch/recovery.rs @@ -10,7 +10,8 @@ use super::{ episode::{ begin_manifest_acceptance_episode, begin_requalified_manifest_acceptance_episode, committed_resource_evidence, complete_manifest_acceptance_episode, hold_uncommitted_manifest_acceptance_episode, - manifest_acceptance_episode_snapshot, manifest_acceptance_exhaustion_reason, manifest_candidate_observation, + manifest_acceptance_episode_snapshot, manifest_acceptance_exhaustion_reason, + manifest_acceptance_generation_is_current, manifest_candidate_observation, record_full_manifest_acceptance_burst, record_manifest_acceptance_exhaustion, record_manifest_acceptance_landscape, update_manifest_acceptance_episode_state, }, @@ -46,6 +47,7 @@ use crate::{ manifest_origin_binding::HlsManifestOriginBinding, recovery_timing::HlsAcceptanceDeadlineMs, }; +use log::debug; use shared::model::{HlsManifestRecoveryBurstLevel, HlsManifestRecoveryBurstPlan}; use std::{ future::Future, @@ -56,7 +58,7 @@ use tokio::task::JoinSet; enum HlsManifestRecoveryAttemptError { Fetch(OriginManifestFetchError), Rejected(HlsManifestRejectLogReason), - Requalified, + Requalified(HlsManifestAcceptanceGeneration), Committed(T), } @@ -93,10 +95,15 @@ where } let attempts = context.retry_policy.attempt_count(); let mut last_error = OriginManifestFetchError::RetryExhausted; - let mut next_attempt_is_full_plan = true; + let (resume, mut expected_generation) = { + let session = context.session.read().await; + let resume = session.origin_qualification_retry_target().is_some(); + (resume, resume.then_some(session.origin_control.acceptance_generation)) + }; + let mut next_attempt_is_full_plan = !resume; let mut requalifications = 0_u8; - let mut attempt_index = 0_usize; - let mut attempt_limit = attempts; + let mut attempt_index = usize::from(resume); + let mut attempt_limit = if resume { 2 } else { attempts }; let mut completed_candidate_requests = 0_usize; while attempt_index < attempt_limit { // A materially changed landscape is requalified immediately. Once @@ -108,13 +115,15 @@ where let jitter = context.retry_policy.sample_jitter_ms(); context.retry_policy.delay_for_attempt_ms(attempt_index, jitter).unwrap_or_default() }; - let acceptance_deadline = current_acceptance_deadline(context).await; - if !acceptance_attempt_may_start( - next_attempt_is_full_plan, - current_time_millis(), - delay_ms, - acceptance_deadline, - ) { + let acceptance_deadline = current_acceptance_deadline(context, expected_generation).await?; + if !resume + && !acceptance_attempt_may_start( + next_attempt_is_full_plan, + current_time_millis(), + delay_ms, + acceptance_deadline, + ) + { return Err(last_error); } if delay_ms > 0 { @@ -131,30 +140,29 @@ where trigger, current_burst_is_full_plan: next_attempt_is_full_plan, may_requalify: requalifications < HLS_MANIFEST_ACCEPTANCE_MAX_REQUALIFICATIONS_PER_REFRESH, - acceptance_deadline, + requalification_window: if resume && attempt_index == 1 { + HlsManifestRequalificationWindow::ResumedAfterSharedBackoff + } else { + HlsManifestRequalificationWindow::CurrentEpisode(acceptance_deadline) + }, completed_candidate_requests, }; let candidate_requests_in_attempt = recovery_burst_plan(context, next_attempt_is_full_plan).total_candidates(); next_attempt_is_full_plan = false; - match fetch_and_commit_manifest_recovery_attempt(context, attempt_plan, &mut commit).await { + match fetch_and_commit_manifest_recovery_attempt(context, attempt_plan, &mut expected_generation, &mut commit) + .await + { HlsManifestRecoveryAttemptError::Committed(committed) => return Ok(committed), - HlsManifestRecoveryAttemptError::Requalified => { + HlsManifestRecoveryAttemptError::Requalified(generation) => { + // Only an explicitly authorized requalification may transfer chain ownership. + expected_generation = Some(generation); requalifications = requalifications.saturating_add(1); next_attempt_is_full_plan = true; last_error = OriginManifestFetchError::RetryExhausted; attempt_limit = attempt_limit_for_started_requalification(attempt_limit, attempt_index); } HlsManifestRecoveryAttemptError::Rejected(reason) if attempt_index.saturating_add(1) < attempt_limit => { - log_manifest_retry_scheduled( - context, - ManifestRetryLogKind::PinnedHostRecovery, - attempt_index, - attempt_limit, - next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), - Some(&reason), - None, - ) - .await; + log_manifest_recovery_retry(context, attempt_index, attempt_limit, Some(&reason), None).await; reject_reason = Some(reason); last_error = OriginManifestFetchError::RetryExhausted; } @@ -165,16 +173,7 @@ where if is_hls_retryable_manifest_reject_fetch_error(&err) && attempt_index.saturating_add(1) < attempt_limit => { - log_manifest_retry_scheduled( - context, - ManifestRetryLogKind::PinnedHostRecovery, - attempt_index, - attempt_limit, - next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), - None, - Some(&err), - ) - .await; + log_manifest_recovery_retry(context, attempt_index, attempt_limit, None, Some(&err)).await; last_error = err; } HlsManifestRecoveryAttemptError::Fetch(err) => return Err(err), @@ -186,6 +185,25 @@ where Err(last_error) } +async fn log_manifest_recovery_retry( + context: &HlsOriginManifestFetchContext, + attempt_index: usize, + attempt_limit: usize, + reject_reason: Option<&HlsManifestRejectLogReason>, + fetch_error: Option<&OriginManifestFetchError>, +) { + log_manifest_retry_scheduled( + context, + ManifestRetryLogKind::PinnedHostRecovery, + attempt_index, + attempt_limit, + next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), + reject_reason, + fetch_error, + ) + .await; +} + pub(super) fn acceptance_attempt_may_start( current_burst_is_full_plan: bool, now_ms: u64, @@ -198,11 +216,20 @@ pub(super) fn acceptance_attempt_may_start( current_burst_is_full_plan || now_ms.saturating_add(delay_ms) < acceptance_deadline.as_millis_since_epoch() } -async fn current_acceptance_deadline(context: &HlsOriginManifestFetchContext) -> HlsAcceptanceDeadlineMs { - context.session.read().await.origin_control.acceptance_episode.as_ref().map_or_else( +async fn current_acceptance_deadline( + context: &HlsOriginManifestFetchContext, + expected_generation: Option, +) -> Result { + let session = context.session.read().await; + if expected_generation.is_some_and(|generation| !manifest_acceptance_generation_is_current(&session, generation)) { + return Err(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } + Ok(session.origin_control.acceptance_episode.as_ref().map_or_else( || HlsAcceptanceDeadlineMs::from_millis_since_epoch(u64::MAX), |episode| episode.timing().acceptance_deadline, - ) + )) } pub(super) fn attempt_limit_for_started_requalification(attempt_limit: usize, attempt_index: usize) -> usize { @@ -212,6 +239,12 @@ pub(super) fn attempt_limit_for_started_requalification(attempt_limit: usize, at attempt_limit.max(attempt_index.saturating_add(2)) } +/// A due session retry can open a fresh episode only after changed origin evidence. +pub(super) enum HlsManifestRequalificationWindow { + CurrentEpisode(HlsAcceptanceDeadlineMs), + ResumedAfterSharedBackoff, +} + pub(super) struct HlsManifestRecoveryAttemptPlan<'a> { pub(super) binding: &'a HlsManifestOriginBinding, pub(super) attempt_index: usize, @@ -222,13 +255,14 @@ pub(super) struct HlsManifestRecoveryAttemptPlan<'a> { pub(super) trigger: HlsManifestAcceptanceTrigger, current_burst_is_full_plan: bool, may_requalify: bool, - acceptance_deadline: HlsAcceptanceDeadlineMs, + requalification_window: HlsManifestRequalificationWindow, completed_candidate_requests: usize, } async fn fetch_and_commit_manifest_recovery_attempt( context: &HlsOriginManifestFetchContext, plan: HlsManifestRecoveryAttemptPlan<'_>, + expected_generation: &mut Option, commit: &mut C, ) -> HlsManifestRecoveryAttemptError where @@ -236,20 +270,27 @@ where Fut: Future>, { let burst_plan = recovery_burst_plan(context, plan.current_burst_is_full_plan); - fetch_and_commit_manifest_recovery_burst_attempt(context, plan, burst_plan, commit).await + fetch_and_commit_manifest_recovery_burst_attempt(context, plan, burst_plan, expected_generation, commit).await } async fn fetch_and_commit_manifest_recovery_burst_attempt( context: &HlsOriginManifestFetchContext, plan: HlsManifestRecoveryAttemptPlan<'_>, burst_plan: HlsManifestRecoveryBurstPlan, + expected_generation: &mut Option, commit: &mut C, ) -> HlsManifestRecoveryAttemptError where C: FnMut(FetchedOriginManifest, HlsManifestCommitAcceptanceMode) -> Fut, Fut: Future>, { - let episode_generation = begin_manifest_acceptance_episode(context, &plan, burst_plan).await; + let episode_generation = begin_manifest_acceptance_episode(context, &plan, burst_plan, *expected_generation).await; + if episode_generation.is_none() { + return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } + *expected_generation = episode_generation; let HlsManifestRecoveryBurstCollection { fetched_candidates, completed_candidates, @@ -273,14 +314,19 @@ where OriginManifestFetchError::DeterministicTimelineConflict(conflict), ); } - HlsManifestRecoveryBurstEvaluationOutcome::Requalified => { - return HlsManifestRecoveryAttemptError::Requalified; + HlsManifestRecoveryBurstEvaluationOutcome::Requalified(generation) => { + return HlsManifestRecoveryAttemptError::Requalified(generation); } HlsManifestRecoveryBurstEvaluationOutcome::Exhausted => { return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RetryExhausted); } + HlsManifestRecoveryBurstEvaluationOutcome::Superseded => { + return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } }; - match commit_selected_manifest_recovery_candidate( + let selection_resumed = match commit_selected_manifest_recovery_candidate( context, &plan, burst_plan, @@ -295,7 +341,19 @@ where HlsSelectedManifestRecoveryCommit::Committed(committed) => { return HlsManifestRecoveryAttemptError::Committed(committed); } - HlsSelectedManifestRecoveryCommit::Rejected(reason) => last_reject_reason = Some(reason), + HlsSelectedManifestRecoveryCommit::Rejected(reason) => { + last_reject_reason = Some(reason); + true + } + HlsSelectedManifestRecoveryCommit::SelectionRejected(HlsRecoveryWorkloadBindingUpdate::StaleGeneration) => { + return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } + HlsSelectedManifestRecoveryCommit::SelectionRejected(_) => { + last_reject_reason = Some(HlsManifestRejectLogReason::StagedSwitchInvalidated); + false + } HlsSelectedManifestRecoveryCommit::LocalRepresentationLimit(violation) => { return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::LocalRepresentationLimit( violation, @@ -307,22 +365,16 @@ where HlsSelectedManifestRecoveryCommit::CommitGenerationExhausted => { return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::CommitGenerationExhausted); } - HlsSelectedManifestRecoveryCommit::NotSelected => {} - } + HlsSelectedManifestRecoveryCommit::NotSelected => false, + }; - if plan.current_burst_is_full_plan { - record_manifest_acceptance_exhaustion( - context, - episode_generation, - manifest_acceptance_exhaustion_reason(&evaluation.observations), - ) - .await; - } - hold_uncommitted_manifest_acceptance_episode( + hold_manifest_recovery_burst( context, episode_generation, + &evaluation.observations, evaluation.held_alternative, evaluation.next_retry_at_ms, + plan.current_burst_is_full_plan || selection_resumed, ) .await; if let Some(reason) = last_reject_reason { @@ -331,6 +383,25 @@ where HlsManifestRecoveryAttemptError::Fetch(last_fetch_error.unwrap_or(OriginManifestFetchError::RetryExhausted)) } +async fn hold_manifest_recovery_burst( + context: &HlsOriginManifestFetchContext, + episode_generation: Option, + observations: &[HlsManifestCandidateObservation], + held_alternative: Option, + next_retry_at_ms: u64, + record_exhaustion: bool, +) { + if record_exhaustion { + record_manifest_acceptance_exhaustion( + context, + episode_generation, + manifest_acceptance_exhaustion_reason(observations), + ) + .await; + } + hold_uncommitted_manifest_acceptance_episode(context, episode_generation, held_alternative, next_retry_at_ms).await; +} + struct HlsManifestRecoveryBurstEvaluation { observations: Vec, acceptance_plan: HlsManifestCommitPlan, @@ -341,8 +412,9 @@ struct HlsManifestRecoveryBurstEvaluation { enum HlsManifestRecoveryBurstEvaluationOutcome { Continue(Box), DeterministicConflict(Box), - Requalified, + Requalified(HlsManifestAcceptanceGeneration), Exhausted, + Superseded, } const fn manifest_acceptance_state_for_plan(plan: &HlsManifestCommitPlan) -> HlsManifestAcceptanceState { @@ -396,6 +468,10 @@ async fn evaluate_manifest_recovery_burst( } } let episode_snapshot = manifest_acceptance_episode_snapshot(context, episode_generation).await; + if episode_snapshot.is_none() { + // An old response must not let the next attempt join a successor episode. + return HlsManifestRecoveryBurstEvaluationOutcome::Superseded; + } let reduced_landscape_changed = !plan.current_burst_is_full_plan && episode_snapshot .as_ref() @@ -403,16 +479,17 @@ async fn evaluate_manifest_recovery_burst( .map(|landscape| classify_reduced_retry_landscape(landscape, &observations)) .is_some_and(HlsReducedRetryLandscapeChange::requires_full_requalification); if reduced_landscape_changed { - if plan.may_requalify - && begin_requalified_manifest_acceptance_episode( + if plan.may_requalify { + if let Some(generation) = begin_requalified_manifest_acceptance_episode( context, episode_generation, plan.trigger, - plan.acceptance_deadline, + &plan.requalification_window, ) .await - { - return HlsManifestRecoveryBurstEvaluationOutcome::Requalified; + { + return HlsManifestRecoveryBurstEvaluationOutcome::Requalified(generation); + } } // A changed landscape may never fall through into reduced-burst // cross-host acceptance when its full requalification budget is gone. @@ -458,6 +535,7 @@ async fn evaluate_manifest_recovery_burst( enum HlsSelectedManifestRecoveryCommit { Committed(T), Rejected(HlsManifestRejectLogReason), + SelectionRejected(HlsRecoveryWorkloadBindingUpdate), LocalRepresentationLimit(crate::manifest_limits::HlsManifestLimitViolation), MalformedTransientRepresentation, CommitGenerationExhausted, @@ -493,8 +571,15 @@ where report.quality.effective_host.as_deref(), &fetched.body, ); - if !select_manifest_recovery_candidate(context, episode_generation, candidate_identity).await { - return HlsSelectedManifestRecoveryCommit::Rejected(HlsManifestRejectLogReason::StagedSwitchInvalidated); + let selection_update = select_manifest_recovery_candidate( + context, + episode_generation, + candidate_identity, + plan.current_burst_is_full_plan, + ) + .await; + if selection_update != HlsRecoveryWorkloadBindingUpdate::Applied { + return HlsSelectedManifestRecoveryCommit::SelectionRejected(selection_update); } let acceptance_mode = selected_commit_acceptance_mode(selected_commit_kind, plan.acceptance_mode); let selection = if burst_plan.total_candidates() > 1 { @@ -558,15 +643,36 @@ async fn select_manifest_recovery_candidate( context: &HlsOriginManifestFetchContext, generation: Option, candidate_identity: HlsManifestRecoveryCandidateIdentity, -) -> bool { + current_burst_is_full_plan: bool, +) -> HlsRecoveryWorkloadBindingUpdate { + let attempt_type = if current_burst_is_full_plan { "full" } else { "reduced" }; let Some(generation) = generation else { - return false; + let session = context.session.read().await; + debug!("HLS manifest candidate selection rejected: update=EpisodeInactive attempt_type={attempt_type} expected_generation=none current_generation={} episode_status={:?}", + session.origin_control.acceptance_generation.0, + crate::manifest_acceptance::manifest_acceptance_episode_status(session.origin_control.acceptance_episode.as_ref(), session.origin_control.acceptance_generation, current_time_millis())); + return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; }; let mut session = context.session.write().await; + let current_generation = session.origin_control.acceptance_generation; let Some(episode) = session.origin_control.acceptance_episode.as_mut() else { - return false; + debug!("HLS manifest candidate selection rejected: update=StaleGeneration attempt_type={attempt_type} expected_generation={} current_generation={} episode_status=Missing", generation.0, current_generation.0); + return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; }; - episode.select_candidate(generation, candidate_identity) == HlsRecoveryWorkloadBindingUpdate::Applied + let update = if current_burst_is_full_plan { + episode.select_candidate(generation, candidate_identity) + } else { + episode.select_candidate_for_reduced_retry(generation, candidate_identity) + }; + if update != HlsRecoveryWorkloadBindingUpdate::Applied { + debug!( + "HLS manifest candidate selection rejected: update={update:?} attempt_type={attempt_type} expected_generation={} current_generation={} episode_status={:?}", + generation.0, + episode.generation.0, + crate::manifest_acceptance::manifest_acceptance_episode_status(Some(episode), current_generation, current_time_millis()), + ); + } + update } const fn selected_commit_acceptance_mode( diff --git a/backend/hls/src/manifest_fetch/tests.rs b/backend/hls/src/manifest_fetch/tests.rs index 69791fe37..e25f6731d 100644 --- a/backend/hls/src/manifest_fetch/tests.rs +++ b/backend/hls/src/manifest_fetch/tests.rs @@ -8,6 +8,7 @@ use super::{ episode::candidate_resource_timeline_evidence, error::{HlsManifestRejectLogReason, OriginManifestFetchError}, + fetch::cap_manifest_retry_after_ms, fingerprint::{ build_manifest_timeline_fingerprint, deterministic_conflict_fingerprint, deterministic_timeline_conflict_from_rejection, @@ -28,6 +29,7 @@ use crate::{ recovery_timing::HlsAcceptanceDeadlineMs, resource_identity::{HlsMediaResourceIdentity, HlsMediaResourceSemanticKey}, timeline::HlsResourceReplayDecision, + HlsSession, HlsSessionKey, }; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use flate2::{ @@ -384,6 +386,38 @@ fn initial_manifest_retry_delay_uses_next_slot_bounded_jitter_and_retry_after_ov assert_eq!(super::next_retry_delay_ms(&retry_policy, 0, Some(3_000), 25), 3_000); } +#[test] +fn manifest_retry_after_is_capped_at_half_the_origin_target_duration() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_control.record_media_progress(50, 12_000); + session.target_duration = Some(2); + + assert_eq!( + [0, 3_000, 6_000, 60_000, u64::MAX].map(|delay_ms| cap_manifest_retry_after_ms(&session, delay_ms)), + [0, 3_000, 6_000, 6_000, 6_000] + ); +} + +#[test] +fn manifest_retry_after_uses_session_target_duration_when_origin_timing_is_missing() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.target_duration = Some(3); + + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 1_500); + session.origin_control.target_duration_snapshot_ms = Some(0); + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 1_500); +} + +#[test] +fn manifest_retry_after_uses_default_target_duration_before_first_manifest() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 7_500); + session.target_duration = Some(0); + session.origin_control.target_duration_snapshot_ms = Some(0); + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 7_500); +} + #[test] fn initial_manifest_retry_status_is_not_labeled_as_recovery() { let error = OriginManifestFetchError::Timeout; diff --git a/backend/hls/src/map_fetcher.rs b/backend/hls/src/map_fetcher.rs index 8aa790b1b..5f101c3eb 100644 --- a/backend/hls/src/map_fetcher.rs +++ b/backend/hls/src/map_fetcher.rs @@ -1,10 +1,13 @@ use super::{ - begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, finish_hls_origin_account_io, - hls_object_body_deadline, run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, - HlsAccessLeaseStore, HlsBoundAccountAcquireErrorKind, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, - HlsOriginIoContext, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, HlsOriginResourceFetchError, - HlsOriginResourceFetchTarget, HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSessionHandle, - MapCacheKey, MapCacheStatus, OriginMapFetchRef, ProxyMapId, SegmentFetchPolicy, + build_hls_origin_resource_headers, hls_object_body_deadline, + run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, HlsAccessLeaseStore, + HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchKind, HlsResourceFetchSource, + HlsSegmentCache, HlsSessionHandle, MapCacheKey, MapCacheStatus, OriginMapFetchRef, ProxyMapId, SegmentFetchPolicy, +}; +use crate::origin_attempt::{ + prepare_media_origin_attempt, HlsMediaOriginAttemptGuard as MapOriginAttemptGuard, + HlsMediaOriginWorkFinish as MapOriginWorkFinish, }; use arc_swap::ArcSwap; use axum::http::HeaderMap; @@ -43,12 +46,6 @@ struct MapFetchCommit { generation_valid: bool, } -#[derive(Clone, Copy)] -struct MapOriginWorkFinish { - generation_valid: bool, - refresh_reservation: bool, -} - impl fmt::Debug for MapFetchSnapshot { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("MapFetchSnapshot") @@ -148,11 +145,18 @@ impl HlsMapWorkerPool { now_ms: u64, policy: &SegmentFetchPolicy, ) -> Option { - let (proxy_session_id, gc_marked_for_removal) = { + let (proxy_session_id, gc_marked_for_removal, needs_origin_io) = { let session = context.session.read().await; - (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) + ( + session.proxy_session_id.clone(), + session.is_gc_marked_for_removal(), + session.maps.values().any(|entry| { + matches!(entry.status, MapCacheStatus::Discovered | MapCacheStatus::Queued { .. }) + && entry.origin_fetch_ref.is_some() + }), + ) }; - if gc_marked_for_removal { + if gc_marked_for_removal || !needs_origin_io { return None; } if !self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms) { @@ -164,6 +168,12 @@ impl HlsMapWorkerPool { } return None; } + if crate::origin_account_runtime::prepare_media_origin(context.origin_io.as_ref(), &context.session) + .await + .is_err() + { + return None; + } let mut session = context.session.write().await; if session.is_gc_marked_for_removal() { return None; @@ -173,7 +183,9 @@ impl HlsMapWorkerPool { } let proxy_map_id = session.maps.iter().find_map(|(proxy_map_id, entry)| { - matches!(entry.status, MapCacheStatus::Discovered | MapCacheStatus::Queued { .. }).then_some(*proxy_map_id) + (matches!(entry.status, MapCacheStatus::Discovered | MapCacheStatus::Queued { .. }) + && entry.origin_fetch_ref.as_ref().is_some_and(|fetch_ref| fetch_ref.is_valid_at(now_ms))) + .then_some(*proxy_map_id) })?; let origin_work_generation = session.activity.origin_work_generation; @@ -254,6 +266,7 @@ impl HlsMapWorkerPool { } } else if entry.cache_key == snapshot.cache_key && matches!(entry.status, MapCacheStatus::Fetching { .. }) + && !fetch_succeeded { entry.status = MapCacheStatus::Discovered; } @@ -277,6 +290,12 @@ impl HlsMapWorkerPool { if let Err(err) = context.segment_cache.delete(&snapshot.cache_key).await { debug!("HLS stale map cache cleanup failed: resource=map/{} error={err}", snapshot.proxy_map_id_log); } + let mut session = context.session.write().await; + if let Some(entry) = session.maps.get_mut(&snapshot.proxy_map_id).filter(|entry| { + entry.cache_key == snapshot.cache_key && matches!(entry.status, MapCacheStatus::Fetching { .. }) + }) { + entry.status = MapCacheStatus::Discovered; + } } if let (Some(coordinator), Some(proxy_session_id)) = (self.availability_reevaluations.as_ref(), evidence_changed_for.as_ref()) @@ -284,7 +303,8 @@ impl HlsMapWorkerPool { coordinator.notify_session_evidence_changed(proxy_session_id); } drop(permit); - if generation_valid { + if generation_valid || context.session.read().await.origin_handoff_ready_after(snapshot.origin_work_generation) + { self.schedule_wake(context, finished_at_ms); } } @@ -309,95 +329,23 @@ async fn fetch_map_into_cache( fetch_map_with_retries_into_cache(context, snapshot, policy).await } -struct MapOriginAttemptGuard { - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -} - async fn prepare_map_origin_attempt( context: MapFetchContext, policy: SegmentFetchPolicy, + expected_generation: u64, ) -> Result { - let started_generation = start_map_origin_work(&context).await; - let binding = - if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; - let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { - if binding.is_detached() { - finish_map_origin_work(&context, started_generation).await; - touch_map_origin_account_binding(&context, false).await; - return Err(MapFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); - } - let guard = match begin_hls_origin_account_io_bounded( - origin_io, - &context.session, - binding, - hls_object_body_deadline(policy.origin_segment_timeout_ms), - ) - .await - { - Ok(guard) => guard, - Err(err) => { - finish_map_origin_work(&context, started_generation).await; - touch_map_origin_account_binding(&context, false).await; - return Err(MapFetchError::ProviderUnavailable(err)); - } - }; - Some((origin_io.clone(), guard)) - } else { - None - }; - Ok(MapOriginAttemptGuard { started_generation, provider_lease }) + prepare_media_origin_attempt( + context.origin_io.as_ref(), + &context.session, + expected_generation, + hls_object_body_deadline(policy.origin_segment_timeout_ms), + ) + .await + .map_err(MapFetchError::ProviderUnavailable) } async fn finish_map_origin_attempt(context: MapFetchContext, guard: MapOriginAttemptGuard) -> MapOriginWorkFinish { - finish_map_origin_io(&context, guard.started_generation, guard.provider_lease).await -} - -async fn finish_map_origin_io( - context: &MapFetchContext, - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -) -> MapOriginWorkFinish { - let origin_work = finish_map_origin_work(context, started_generation).await; - if let Some((origin_io, guard)) = provider_lease { - finish_hls_origin_account_io( - &origin_io, - &context.session, - guard, - origin_work.generation_valid && origin_work.refresh_reservation, - ) - .await; - touch_map_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) - .await; - } - origin_work -} - -async fn start_map_origin_work(context: &MapFetchContext) -> Option { - context.origin_io.as_ref()?; - let mut session = context.session.write().await; - Some(session.start_origin_work()) -} - -async fn finish_map_origin_work(context: &MapFetchContext, started_generation: Option) -> MapOriginWorkFinish { - let Some(started_generation) = started_generation else { - return MapOriginWorkFinish { generation_valid: true, refresh_reservation: false }; - }; - let mut session = context.session.write().await; - let generation_valid = session.finish_origin_work(started_generation); - let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); - MapOriginWorkFinish { generation_valid, refresh_reservation } -} - -async fn touch_map_origin_account_binding(context: &MapFetchContext, reservation_refreshed: bool) { - let mut session = context.session.write().await; - if let Some(binding) = session.origin_account_binding.as_mut() { - let now_ms = current_time_millis(); - binding.last_origin_io_at_ms = Some(now_ms); - if reservation_refreshed { - binding.last_reservation_refresh_at_ms = Some(now_ms); - } - } + guard.finish(&context.session).await } #[allow(clippy::too_many_lines)] @@ -406,11 +354,8 @@ async fn fetch_map_with_retries_into_cache( snapshot: &MapFetchSnapshot, policy: &SegmentFetchPolicy, ) -> Result { - let headers = build_map_origin_headers( - &context.headers, - &context.origin_provider_session_headers, - snapshot.fetch_ref.byte_range, - )?; + let provider_headers = context.session.read().await.origin_provider_session_headers.clone(); + let headers = build_map_origin_headers(&context.headers, &provider_headers, snapshot.fetch_ref.byte_range)?; let target = HlsOriginResourceFetchTarget { kind: HlsResourceFetchKind::Map, source: HlsResourceFetchSource::Normal, @@ -432,6 +377,7 @@ async fn fetch_map_with_retries_into_cache( let session = context.session.read().await; super::HlsLogIdentity::from_session(&session) }; + let expected_generation = snapshot.origin_work_generation; let context = context.clone(); let snapshot = snapshot.clone(); let policy_for_prepare = policy.clone(); @@ -445,7 +391,7 @@ async fn fetch_map_with_retries_into_cache( move |_attempt| { let context = prepare_context.clone(); let policy = policy_for_prepare.clone(); - async move { prepare_map_origin_attempt(context, policy).await }.boxed() + async move { prepare_map_origin_attempt(context, policy, expected_generation).await }.boxed() }, move |guard| { let context = cleanup_context.clone(); @@ -570,6 +516,78 @@ mod tests { assert!(!headers.contains_key(header::HOST)); } + #[tokio::test] + async fn account_handoff_during_map_prepare_blocks_old_origin_request() { + use crate::{origin_attempt::tests::bound_origin_io, HlsOriginAccountIoLease, HlsOriginResourceFetchError}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("origin address"); + let requests = Arc::new(AtomicUsize::new(0)); + let seen = Arc::clone(&requests); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("origin request"); + let mut request = [0_u8; 2048]; + let read = socket.read(&mut request).await.expect("request reads"); + assert!(read > 0, "origin receives a request"); + seen.fetch_add(1, Ordering::AcqRel); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nmap") + .await + .expect("response writes"); + }); + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "map-handoff"), b"secret", 0).await; + let (io, replacement) = bound_origin_io(&session).await; + { + let mut session = session.write().await; + session + .apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n", + &format!("http://{addr}/account-a/index.m3u8"), + )) + .expect("manifest maps"); + session.queue_map_fetch_candidates(1); + } + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let worker = HlsMapWorkerPool::new(policy.clone()); + grant_usable_map_access_lease(&worker, &session.read().await.proxy_session_id).await; + let context = MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: Some(io), + }; + let snapshot = worker.next_fetch_snapshot(&context, 2, &policy).await.expect("map snapshot"); + let replacement_lease = Arc::new(HlsOriginAccountIoLease::active_for_test(&replacement, 1)); + let read_blocker = session.read().await; + let mut fetch = Box::pin(super::fetch_map_with_retries_into_cache(&context, &snapshot, &policy)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + let mut handoff = Box::pin(session.write()); + assert!(futures::poll!(handoff.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + { + let mut session = handoff.await; + session.handoff_origin_account(replacement); + session.origin_account_io_lease = Some(Arc::clone(&replacement_lease)); + } + + let result = fetch.await; + server.abort(); + assert_eq!(requests.load(Ordering::Acquire), 0, "stale MAP URL must never reach the origin"); + assert!(matches!(result, Err(HlsOriginResourceFetchError::ProviderUnavailable(_)))); + let session = session.read().await; + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert_eq!(replacement_lease.active_io_count.load(Ordering::Acquire), 1); + } + #[tokio::test] async fn map_fetch_writes_cache_and_sets_ready_after_commit() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); diff --git a/backend/hls/src/media_reserve.rs b/backend/hls/src/media_reserve.rs index 4ef0849d9..cd482f9a5 100644 --- a/backend/hls/src/media_reserve.rs +++ b/backend/hls/src/media_reserve.rs @@ -3,7 +3,7 @@ use super::{ terminal_tail::{HlsEncryptionSignature, HlsMapSignature, HlsMediaContainer}, HlsAccessLeaseId, TransientManifestGeneration, HLS_ACCESS_LEASE_ID_PLACEHOLDER, }; -use std::{borrow::Cow, collections::BTreeSet, sync::Arc}; +use std::{borrow::Cow, collections::BTreeMap, sync::Arc}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct HlsLeaseManifestSegment { @@ -121,7 +121,9 @@ pub struct HlsLeasePlaybackCursor { pub cursor_generation: u64, request_epoch: u64, highest_requested_proxy_seq: Option, - out_of_order_completed_proxy_seqs: BTreeSet, + first_requested_media_start_ms: Option, + highest_contiguous_completed_media_end_ms: Option, + out_of_order_completed_proxy_seqs: BTreeMap>, } const MAX_OUT_OF_ORDER_COMPLETIONS: usize = 64; @@ -132,6 +134,7 @@ const MAX_OUT_OF_ORDER_COMPLETIONS: usize = 64; pub struct HlsPlaybackRequestToken { proxy_seq: u64, request_epoch: u64, + media_end_ms: Option, } /// Result of committing a fully delivered segment to a playback cursor. @@ -146,9 +149,16 @@ pub enum HlsPlaybackCompletionOutcome { } impl HlsLeasePlaybackCursor { - /// Earliest sequence whose duration is needed to align the current manifest - /// window with this lease's measured playback position. + /// Earliest sequence still needed for playback evidence. Completed history is + /// carried by the cursor's media clock; undelivered gaps must remain visible. pub fn ready_timeline_start_proxy_seq(&self, manifest_first_proxy_seq: u64) -> u64 { + if self.first_requested_media_start_ms.is_some() { + return self + .highest_contiguous_completed_proxy_seq + .and_then(|seq| seq.checked_add(1)) + .or(self.first_requested_proxy_seq) + .map_or(manifest_first_proxy_seq, |seq| seq.min(manifest_first_proxy_seq)); + } self.first_requested_proxy_seq.map_or(manifest_first_proxy_seq, |first_requested_proxy_seq| { first_requested_proxy_seq.min(manifest_first_proxy_seq) }) @@ -164,6 +174,8 @@ impl HlsLeasePlaybackCursor { self.first_segment_completed_at_ms = None; self.highest_contiguous_completed_proxy_seq = None; self.highest_requested_proxy_seq = Some(proxy_seq); + self.first_requested_media_start_ms = None; + self.highest_contiguous_completed_media_end_ms = None; self.out_of_order_completed_proxy_seqs.clear(); } else { self.first_requested_proxy_seq.get_or_insert(proxy_seq); @@ -177,7 +189,22 @@ impl HlsLeasePlaybackCursor { ); self.cursor_generation = self.cursor_generation.saturating_add(1); - HlsPlaybackRequestToken { proxy_seq, request_epoch: self.request_epoch } + HlsPlaybackRequestToken { proxy_seq, request_epoch: self.request_epoch, media_end_ms: None } + } + + /// Binds playback evidence to the immutable session media clock before cache entries can be collected. + pub(crate) fn bind_request_timing( + &mut self, + mut token: HlsPlaybackRequestToken, + segment: &crate::SegmentEntry, + ) -> HlsPlaybackRequestToken { + debug_assert_eq!(token.proxy_seq, segment.proxy_seq); + debug_assert_eq!(token.request_epoch, self.request_epoch); + if self.first_requested_proxy_seq == Some(token.proxy_seq) { + self.first_requested_media_start_ms.get_or_insert(segment.media_start_ms); + } + token.media_end_ms = Some(segment.media_start_ms.saturating_add(segment.duration_ms)); + token } /// Commits a token only after the corresponding response body was delivered completely. @@ -196,7 +223,7 @@ impl HlsLeasePlaybackCursor { return HlsPlaybackCompletionOutcome::BeforeCurrentPosition; } if self.highest_contiguous_completed_proxy_seq.is_some_and(|highest| token.proxy_seq <= highest) - || self.out_of_order_completed_proxy_seqs.contains(&token.proxy_seq) + || self.out_of_order_completed_proxy_seqs.contains_key(&token.proxy_seq) { return HlsPlaybackCompletionOutcome::Duplicate; } @@ -206,7 +233,7 @@ impl HlsLeasePlaybackCursor { .and_then(|highest| highest.checked_add(1)) .unwrap_or(first_requested_proxy_seq); if token.proxy_seq != next_expected_proxy_seq { - self.buffer_out_of_order_completion(token.proxy_seq); + self.buffer_out_of_order_completion(token); self.cursor_generation = self.cursor_generation.saturating_add(1); return HlsPlaybackCompletionOutcome::Buffered; } @@ -215,29 +242,29 @@ impl HlsLeasePlaybackCursor { self.first_segment_completed_at_ms = Some(completed_at_ms); } self.highest_contiguous_completed_proxy_seq = Some(token.proxy_seq); + self.highest_contiguous_completed_media_end_ms = token.media_end_ms; self.advance_over_buffered_completions(); self.cursor_generation = self.cursor_generation.saturating_add(1); HlsPlaybackCompletionOutcome::Advanced } - fn buffer_out_of_order_completion(&mut self, proxy_seq: u64) { - self.out_of_order_completed_proxy_seqs.insert(proxy_seq); + fn buffer_out_of_order_completion(&mut self, token: HlsPlaybackRequestToken) { + self.out_of_order_completed_proxy_seqs.insert(token.proxy_seq, token.media_end_ms); if self.out_of_order_completed_proxy_seqs.len() <= MAX_OUT_OF_ORDER_COMPLETIONS { return; } - if let Some(furthest_proxy_seq) = self.out_of_order_completed_proxy_seqs.last().copied() { - self.out_of_order_completed_proxy_seqs.remove(&furthest_proxy_seq); - } + self.out_of_order_completed_proxy_seqs.pop_last(); } fn advance_over_buffered_completions(&mut self) { while let Some(next_proxy_seq) = self.highest_contiguous_completed_proxy_seq.and_then(|highest| highest.checked_add(1)) { - if !self.out_of_order_completed_proxy_seqs.remove(&next_proxy_seq) { + let Some(media_end_ms) = self.out_of_order_completed_proxy_seqs.remove(&next_proxy_seq) else { break; - } + }; self.highest_contiguous_completed_proxy_seq = Some(next_proxy_seq); + self.highest_contiguous_completed_media_end_ms = media_end_ms; } } } @@ -311,7 +338,12 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv cutover_required: input.origin_path_degraded && !input.recovery_committed, }; } - let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest); + let first_needed_seq = if input.cursor.first_requested_media_start_ms.is_some() { + input.cursor.ready_timeline_start_proxy_seq(input.manifest.first_proxy_seq) + } else { + input.manifest.first_proxy_seq + }; + let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest, first_needed_seq); let visible_tail = aligned_visible_tail_ms(input.ready_timeline, input.manifest); let guaranteed_media_horizon_ms = guaranteed_ready_span.horizon_ms; let conservative_playback_position_ms = @@ -353,9 +385,11 @@ struct HlsGuaranteedReadySpan { fn contiguous_ready_span( timeline: &HlsReadyTimelineSnapshot, manifest: &HlsLeaseManifestSnapshot, + first_proxy_seq: u64, ) -> HlsGuaranteedReadySpan { - let fallback_start_ms = manifest_visible_start_ms(manifest); - let Some(start_index) = timeline.units.iter().position(|unit| unit.proxy_seq == manifest.first_proxy_seq) else { + let fallback_start_ms = + if first_proxy_seq == manifest.first_proxy_seq { manifest_visible_start_ms(manifest) } else { 0 }; + let Some(start_index) = timeline.units.iter().position(|unit| unit.proxy_seq == first_proxy_seq) else { return HlsGuaranteedReadySpan { start_ms: fallback_start_ms, horizon_ms: fallback_start_ms, @@ -376,7 +410,7 @@ fn contiguous_ready_span( }; } - let mut expected_seq = manifest.first_proxy_seq; + let mut expected_seq = first_proxy_seq; let mut expected_start_ms = first_unit.start_ms; let mut horizon_ms = first_unit.start_ms; let mut end_unit_index = start_index; @@ -436,21 +470,29 @@ fn conservative_playback_position( let Some(first_seq) = input.cursor.first_requested_proxy_seq else { return visible_tail_ms; }; - let Some(first_unit) = input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == first_seq) else { + let Some(first_start_ms) = input.cursor.first_requested_media_start_ms.or_else(|| { + input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == first_seq).map(|unit| unit.start_ms) + }) else { return visible_tail_ms.max(guaranteed_media_horizon_ms); }; if input.cursor.first_segment_completed_at_ms.is_none() { - return visible_tail_ms.max(first_unit.start_ms); + return visible_tail_ms.max(first_start_ms); } let delivered_end = input .cursor - .highest_contiguous_completed_proxy_seq - .and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq)) - .map_or(guaranteed_media_horizon_ms, |unit| unit.start_ms.saturating_add(unit.duration_ms)); + .highest_contiguous_completed_media_end_ms + .or_else(|| { + input + .cursor + .highest_contiguous_completed_proxy_seq + .and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq)) + .map(|unit| unit.start_ms.saturating_add(unit.duration_ms)) + }) + .unwrap_or(guaranteed_media_horizon_ms); let elapsed_guarded = input.cursor.first_segment_completed_at_ms.map_or(0, |completed_at_ms| { input.now_ms.saturating_sub(completed_at_ms).saturating_mul(u64::from(input.playback_rate_guard_milli)) / 1_000 }); - delivered_end.min(first_unit.start_ms.saturating_add(elapsed_guarded)) + delivered_end.min(first_start_ms.saturating_add(elapsed_guarded)) } fn guaranteed_transition_margin( @@ -539,7 +581,8 @@ pub fn evaluate_startup_admission(input: HlsStartupAdmissionInput<'_>) -> HlsSta decision, }; } - let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest); + let guaranteed_ready_span = + contiguous_ready_span(input.ready_timeline, input.manifest, input.manifest.first_proxy_seq); let visible_tail_ms = aligned_visible_tail_ms(input.ready_timeline, input.manifest); let visible_contiguous_ready_duration_ms = guaranteed_ready_span.horizon_ms.min(visible_tail_ms).saturating_sub(guaranteed_ready_span.start_ms); @@ -693,6 +736,31 @@ mod tests { }) } + #[test] + fn one_and_two_segment_playlists_admit_only_actual_ready_duration_and_claim_no_hidden_reserve() { + for count in [1, 2] { + let mut short = manifest(); + short.visible_segments = Arc::from(&short.visible_segments[..count]); + short.last_proxy_seq = u64::try_from(count - 1).unwrap(); + short.playlist_duration_ms = short.visible_segments.iter().map(|segment| segment.duration_ms).sum(); + short.last_visible_media_end_ms = short.playlist_duration_ms; + let ready = ready_timeline(); + let mut timeline = HlsReadyTimelineSnapshot { units: Arc::from(&ready.units[..count]) }; + let admitted = admission(&short, &timeline, HlsStartupAdmissionOriginState::Healthy, 0); + assert_eq!(admitted.decision, HlsStartupAdmissionDecision::Admit); + assert_eq!(admitted.visible_contiguous_ready_duration_ms, short.playlist_duration_ms); + assert_eq!(admitted.prospective_lease_reserve_ms, 0); + let reserve = reserve(&short, &HlsLeasePlaybackCursor::default(), &timeline, 0); + assert_eq!(reserve.initial_hidden_ready_duration_ms, 0); + assert_eq!(reserve.guaranteed_media_horizon_ms, short.playlist_duration_ms); + Arc::make_mut(&mut timeline.units)[count - 1].state = HlsReadyMediaState::NotReady; + assert_eq!( + admission(&short, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Reject(HlsStartupAdmissionRejection::InsufficientVisibleReadyMedia) + ); + } + } + #[test] fn strip_reserve_is_sum_of_actual_hidden_ready_durations() { let reserve = reserve(&manifest(), &HlsLeasePlaybackCursor::default(), &ready_timeline(), 0); @@ -1211,4 +1279,98 @@ mod tests { HlsStartupAdmissionDecision::Reject(HlsStartupAdmissionRejection::InsufficientVisibleReadyMedia) ); } + + fn variable_duration_session() -> crate::HlsSession { + use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + let OriginManifestParseOutcome::Normal(origin) = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:9\n#EXT-X-MEDIA-SEQUENCE:0\n\ + #EXTINF:4,\n0.ts\n#EXTINF:5,\n1.ts\n#EXTINF:6,\n2.ts\n#EXTINF:3,\n3.ts\n#EXTINF:9,\n4.ts\n", + "http://origin.example/index.m3u8", + ) else { + panic!("variable timeline parses"); + }; + let mut session = crate::HlsSession::new(crate::HlsSessionKey::new(1, "variable-clock"), b"secret", 0); + session.apply_origin_manifest(&origin).expect("variable timeline maps"); + for segment in session.segments.values_mut() { + segment.status = crate::SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 0 }; + } + session + } + + #[test] + fn timed_out_of_order_completions_preserve_variable_duration_anchor_after_collection() { + let mut session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let tokens = (0..3) + .map(|seq| { + let token = cursor.record_request_started(seq, 1_000); + cursor.bind_request_timing(token, &session.segments[&seq]) + }) + .collect::>(); + assert_eq!(cursor.record_request_completed(tokens[2], 1_100), HlsPlaybackCompletionOutcome::Buffered); + assert_eq!(cursor.record_request_completed(tokens[0], 1_200), HlsPlaybackCompletionOutcome::Advanced); + assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(4_000)); + assert_eq!(cursor.record_request_completed(tokens[1], 1_300), HlsPlaybackCompletionOutcome::Advanced); + assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(15_000)); + let mut current = manifest(); + current.first_proxy_seq = 2; + current.last_proxy_seq = 2; + current.visible_segments = Arc::from([current.visible_segments[2].clone()]); + current.playlist_duration_ms = 6_000; + current.last_visible_media_end_ms = 6_000; + let before = reserve(¤t, &cursor, &session.ready_timeline_snapshot(2, 11_200), 11_200); + session.segments.remove(&0); + session.segments.remove(&1); + let after = reserve( + ¤t, + &cursor, + &session.ready_timeline_snapshot(cursor.ready_timeline_start_proxy_seq(current.first_proxy_seq), 11_200), + 11_200, + ); + assert_eq!(before, after); + assert_eq!(after.conservative_playback_position_ms, 10_500); + assert_eq!(after.guaranteed_reserve_ms, 16_500); + } + + #[test] + fn timed_cursor_does_not_bridge_an_undelivered_gap_before_the_new_manifest_window() { + let mut session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let first = cursor.record_request_started(0, 1_000); + let first = cursor.bind_request_timing(first, &session.segments[&0]); + assert_eq!(cursor.record_request_completed(first, 1_000), HlsPlaybackCompletionOutcome::Advanced); + let mut current = manifest(); + current.first_proxy_seq = 2; + current.last_proxy_seq = 2; + current.visible_segments = Arc::from([current.visible_segments[2].clone()]); + current.playlist_duration_ms = 6_000; + current.last_visible_media_end_ms = 6_000; + assert_eq!(cursor.ready_timeline_start_proxy_seq(2), 1); + session.segments.remove(&0); + let timeline = session.ready_timeline_snapshot(1, 2_000); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 25_950); + session.segments.get_mut(&1).expect("undelivered segment").status = crate::SegmentCacheStatus::Discovered; + let timeline = session.ready_timeline_snapshot(1, 2_000); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 2_950); + session.segments.remove(&1); + let timeline = session.ready_timeline_snapshot(1, 2_000); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 0); + } + + #[test] + fn timed_forward_seek_discards_the_previous_media_anchor_and_late_completion() { + let session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let old = cursor.record_request_started(0, 1_000); + let old = cursor.bind_request_timing(old, &session.segments[&0]); + let seek = cursor.record_request_started(4, 2_000); + let seek = cursor.bind_request_timing(seek, &session.segments[&4]); + assert_eq!(cursor.record_request_completed(old, 2_500), HlsPlaybackCompletionOutcome::StaleRequest); + assert_eq!(cursor.record_request_completed(seek, 3_000), HlsPlaybackCompletionOutcome::Advanced); + assert_eq!(cursor.first_requested_media_start_ms, Some(18_000)); + assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(27_000)); + let reserve = reserve(&manifest(), &cursor, &session.ready_timeline_snapshot(0, 4_000), 4_000); + assert_eq!(reserve.conservative_playback_position_ms, 19_050); + assert_eq!(reserve.guaranteed_reserve_ms, 7_950); + } } diff --git a/backend/hls/src/observability.rs b/backend/hls/src/observability.rs index 7d9ae134f..f90367d0f 100644 --- a/backend/hls/src/observability.rs +++ b/backend/hls/src/observability.rs @@ -21,6 +21,10 @@ pub use tuliprox_core::utils::content_coding::{ #[derive(Debug, Default)] pub struct HlsCacheMetrics { revision_disk_bytes: AtomicU64, + revision_deletions_planned: AtomicU64, + revision_deletions_succeeded: AtomicU64, + revision_deletions_failed: AtomicU64, + revision_bytes_reclaimed: AtomicU64, progressive_replay_bytes: AtomicU64, progressive_active_fills: AtomicU64, deferred_repairs: AtomicU64, @@ -61,6 +65,10 @@ pub struct HlsCacheMetrics { #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct HlsCacheMetricsSnapshot { pub revision_disk_bytes: u64, + pub revision_deletions_planned: u64, + pub revision_deletions_succeeded: u64, + pub revision_deletions_failed: u64, + pub revision_bytes_reclaimed: u64, pub progressive_replay_bytes: u64, pub progressive_active_fills: u64, pub deferred_repairs: u64, @@ -104,6 +112,13 @@ impl HlsCacheMetrics { self.progressive_replay_bytes.store(u64::try_from(bytes).unwrap_or(u64::MAX), Ordering::Relaxed); self.deferred_repairs.store(u64::try_from(repairs).unwrap_or(u64::MAX), Ordering::Relaxed); } + pub(crate) fn record_revision_gc(&self, planned: usize, succeeded: usize, failed: usize, bytes: u64) { + self.revision_deletions_planned.fetch_add(u64::try_from(planned).unwrap_or(u64::MAX), Ordering::Relaxed); + self.revision_deletions_succeeded.fetch_add(u64::try_from(succeeded).unwrap_or(u64::MAX), Ordering::Relaxed); + self.revision_deletions_failed.fetch_add(u64::try_from(failed).unwrap_or(u64::MAX), Ordering::Relaxed); + self.revision_bytes_reclaimed.fetch_add(bytes, Ordering::Relaxed); + } + pub fn record_revision_disk_bytes(&self, bytes: u64) { self.revision_disk_bytes.store(bytes, Ordering::Relaxed); } pub fn record_session_created(&self) { increment(&self.sessions_created, 1); } pub fn record_session_reused(&self) { increment(&self.sessions_reused, 1); } @@ -157,6 +172,10 @@ impl HlsCacheMetrics { pub fn snapshot(&self) -> HlsCacheMetricsSnapshot { HlsCacheMetricsSnapshot { revision_disk_bytes: load(&self.revision_disk_bytes), + revision_deletions_planned: load(&self.revision_deletions_planned), + revision_deletions_succeeded: load(&self.revision_deletions_succeeded), + revision_deletions_failed: load(&self.revision_deletions_failed), + revision_bytes_reclaimed: load(&self.revision_bytes_reclaimed), progressive_replay_bytes: load(&self.progressive_replay_bytes), progressive_active_fills: load(&self.progressive_active_fills), deferred_repairs: load(&self.deferred_repairs), diff --git a/backend/hls/src/origin.rs b/backend/hls/src/origin.rs index e18872fc3..7144201ee 100644 --- a/backend/hls/src/origin.rs +++ b/backend/hls/src/origin.rs @@ -227,7 +227,15 @@ impl fmt::Debug for HlsOriginAccountIoLeaseGuard { } impl HlsOriginAccountIoLeaseGuard { - pub fn new(binding: HlsOriginAccountBinding, lease: Arc) -> Self { + pub fn new(mut binding: HlsOriginAccountBinding, lease: Arc) -> Self { + { + let handle = lease.provider_handle.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(handle) = handle.as_ref() { + // A joining caller may still carry the expired predecessor's lease metadata. + binding.playback_request_id = handle.playback_request_id; + binding.provider_binding_tag = handle.binding_tag; + } + } Self { binding, lease, disarmed: false } } @@ -512,12 +520,14 @@ pub enum HlsOriginAccountStatus { Known, Missing, Expired, + Blocked, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HlsBoundAccountAcquireErrorKind { Missing, Expired, + Blocked, Exhausted, ReservedForOther, Detached, @@ -528,7 +538,7 @@ pub enum HlsBoundAccountAcquireErrorKind { } impl HlsBoundAccountAcquireErrorKind { - pub fn allows_rebind(self) -> bool { matches!(self, Self::Missing | Self::Expired) } + pub fn allows_rebind(self) -> bool { matches!(self, Self::Missing | Self::Expired | Self::Blocked) } pub fn is_retryable_resource_failure(self) -> bool { // A competing session or an in-flight binding handoff can temporarily @@ -549,6 +559,7 @@ impl HlsBoundAccountAcquireErrorKind { match self { Self::Missing => "Missing", Self::Expired => "Expired", + Self::Blocked => "Blocked", Self::Exhausted => "Exhausted", Self::ReservedForOther => "ReservedForOther", Self::Detached => "Detached", @@ -634,6 +645,9 @@ pub struct HlsOriginIoContext { pub reservation_ttl_secs: u64, pub preacquired_provider_handle: Option>>>, pub started_generation: Option, + pub prepared_account: Option, + pub playback_kind: PlaybackKind, + pub account_runtime: Option>, } impl HlsOriginIoContext { @@ -649,6 +663,11 @@ impl HlsOriginIoContext { self } + pub fn with_managed_provider_handle(mut self, handle: tuliprox_session::ManagedProviderHandle) -> Self { + self.preacquired_provider_handle = Some(Arc::new(std::sync::Mutex::new(Some(handle)))); + self + } + pub fn take_preacquired_provider_handle(&self) -> Option { let handle = self.preacquired_provider_handle.as_ref()?; let mut guard = match handle.lock() { @@ -673,6 +692,9 @@ pub fn hls_origin_account_status(ctx: &HlsCtx, binding: &HlsOriginAccountBinding if is_input_expired(provider_config.exp_date()) { return HlsOriginAccountStatus::Expired; } + if provider_config.is_account_blocked() { + return HlsOriginAccountStatus::Blocked; + } HlsOriginAccountStatus::Known } @@ -683,6 +705,7 @@ pub fn acquire_bound_hls_origin_account_handle( allow_grace: bool, priority: i8, connection_kind: ConnectionKind, + playback_kind: PlaybackKind, ) -> Result { if binding.is_detached() { return Err(HlsBoundAccountAcquireErrorKind::Detached); @@ -690,6 +713,7 @@ pub fn acquire_bound_hls_origin_account_handle( match hls_origin_account_status(ctx, binding) { HlsOriginAccountStatus::Missing => return Err(HlsBoundAccountAcquireErrorKind::Missing), HlsOriginAccountStatus::Expired => return Err(HlsBoundAccountAcquireErrorKind::Expired), + HlsOriginAccountStatus::Blocked => return Err(HlsBoundAccountAcquireErrorKind::Blocked), HlsOriginAccountStatus::Known => {} } if ctx.active_provider.is_provider_reserved_for_other_session(&binding.account_name, Some(&binding.session_owner)) { @@ -701,7 +725,7 @@ pub fn acquire_bound_hls_origin_account_handle( allow_grace, priority, connection_kind, - Some(PlaybackLeaseRef::new(&binding.session_owner, PlaybackKind::LiveHls)), + Some(PlaybackLeaseRef::new(&binding.session_owner, playback_kind)), ); if let Some(handle) = handle { return Ok(handle); @@ -750,6 +774,13 @@ async fn begin_hls_origin_account_io_inner( let slot = reserve_hls_origin_account_io_slot(session, binding, deadline).await?; if let HlsOriginAccountIoSlot::Joined(lease) = slot { let guard = HlsOriginAccountIoLeaseGuard::new(binding.clone(), lease); + match hls_origin_account_status(&origin_io.ctx, binding) { + HlsOriginAccountStatus::Known => {} + HlsOriginAccountStatus::Missing => return Err(HlsBoundAccountAcquireErrorKind::Missing), + HlsOriginAccountStatus::Expired => return Err(HlsBoundAccountAcquireErrorKind::Expired), + HlsOriginAccountStatus::Blocked => return Err(HlsBoundAccountAcquireErrorKind::Blocked), + } + // Release any unused pre-acquired handle synchronously on drop. It needs no // cleanup permit, so a join cannot self-block on a single-permit cleanup queue. drop(origin_io.take_preacquired_provider_handle()); @@ -766,6 +797,12 @@ async fn begin_hls_origin_account_io_inner( let cleanup_permit = reserve_cleanup_permit(origin_io, deadline).await?; let acquired_handle = if let Some(mut managed) = origin_io.take_preacquired_provider_handle() { + if !managed.handle().is_some_and(|handle| { + handle.allocation.get_provider_name().as_ref() == Some(&binding.account_name) + && binding.provider_binding_tag.is_none_or(|tag| handle.binding_tag == Some(tag)) + }) { + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } // Disarm the managed owner only after the cleanup permit is already reserved, // so the naked handle is immediately paired with its release responsibility. // The wrapper always carries its handle; fall back to a fresh acquire defensively. @@ -778,6 +815,7 @@ async fn begin_hls_origin_account_io_inner( origin_io.allow_grace, origin_io.priority, origin_io.connection_kind, + origin_io.playback_kind, ), } } else { @@ -788,6 +826,7 @@ async fn begin_hls_origin_account_io_inner( origin_io.allow_grace, origin_io.priority, origin_io.connection_kind, + origin_io.playback_kind, ) }; @@ -939,7 +978,7 @@ async fn store_acquired_hls_origin_account_io_handle( pending_guard.disarm(); return Err(HlsBoundAccountAcquireErrorKind::AcquireTimedOut); } - let lease_ref = PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }; + let lease_ref = PlaybackLeaseRef { owner: &binding.session_owner, kind: origin_io.playback_kind, request_id }; origin_io.ctx.active_provider.refresh_playback_lease( &binding.account_name, &lease_ref, @@ -1067,7 +1106,11 @@ pub async fn finish_hls_origin_account_io( if let Some(request_id) = binding.playback_request_id { origin_io.ctx.active_provider.refresh_playback_lease( &binding.account_name, - &PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }, + &PlaybackLeaseRef { + owner: &binding.session_owner, + kind: session.origin_playback_kind(), + request_id, + }, confirmed_ttl_secs, ); } @@ -1347,6 +1390,7 @@ mod tests { fn bound_account_acquire_error_rebind_policy_matches_concept() { assert!(HlsBoundAccountAcquireErrorKind::Missing.allows_rebind()); assert!(HlsBoundAccountAcquireErrorKind::Expired.allows_rebind()); + assert!(HlsBoundAccountAcquireErrorKind::Blocked.allows_rebind()); assert!(!HlsBoundAccountAcquireErrorKind::Exhausted.allows_rebind()); assert!(!HlsBoundAccountAcquireErrorKind::ReservedForOther.allows_rebind()); assert!(!HlsBoundAccountAcquireErrorKind::Detached.allows_rebind()); @@ -1581,6 +1625,9 @@ mod tests { reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, }; let mut session = HlsSession::new(HlsSessionKey::new(1, "permit-timeout"), b"secret", 1_000); diff --git a/backend/hls/src/origin_account_runtime.rs b/backend/hls/src/origin_account_runtime.rs new file mode 100644 index 000000000..23b419cc9 --- /dev/null +++ b/backend/hls/src/origin_account_runtime.rs @@ -0,0 +1,205 @@ +use crate::{ + manifest_fetch::LiveHlsOriginEntry, HlsBoundAccountAcquireErrorKind, HlsOriginIoContext, HlsOriginWorkClass, + HlsSessionHandle, +}; +use std::{fmt, future::Future, pin::Pin}; + +/// Origin operation being prepared; media work must first qualify an account handoff. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HlsOriginAccountWork { + Manifest, + Media, +} + +/// Account preparation result resolved against the current source configuration. +pub struct HlsPreparedOriginAccount { + pub entry: LiveHlsOriginEntry, + pub snapshot: HlsOriginAccountSnapshot, +} + +/// Account identity, work generation and provider headers captured together with a resolved entry URL. +#[derive(Clone)] +pub struct HlsOriginAccountSnapshot { + pub binding: Option, + pub generation: u64, + pub provider_session_headers: axum::http::HeaderMap, +} + +impl HlsOriginAccountSnapshot { + pub fn from_session(session: &crate::HlsSession) -> Self { + Self { + binding: session.origin_account_binding.clone(), + generation: session.activity.origin_work_generation, + provider_session_headers: session.origin_provider_session_headers.clone(), + } + } + + pub fn matches_session(&self, session: &crate::HlsSession) -> bool { + self.generation == session.activity.origin_work_generation + && match (&self.binding, &session.origin_account_binding) { + (Some(expected), Some(current)) => { + expected.input_name == current.input_name + && expected.account_name == current.account_name + && expected.session_owner == current.session_owner + && expected.generation == current.generation + && expected.provider_binding_tag == current.provider_binding_tag + && expected.is_active() == current.is_active() + } + (None, None) => true, + (Some(_), None) | (None, Some(_)) => false, + } + } +} + +/// App-owned account allocation/URL adapter used by existing HLS workers. +pub trait HlsOriginAccountRuntime: Send + Sync + fmt::Debug { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> Pin> + Send + 'a>>; +} + +pub(crate) async fn prepare_media_origin( + origin_io: Option<&HlsOriginIoContext>, + session: &HlsSessionHandle, +) -> Result<(), HlsBoundAccountAcquireErrorKind> { + let Some(io) = origin_io else { + return Ok(()); + }; + let Some(runtime) = io.account_runtime.as_ref() else { + return Ok(()); + }; + let class = if io.allow_grace { HlsOriginWorkClass::Demand } else { HlsOriginWorkClass::Background }; + let deadline = tokio::time::Instant::now() + io.ctx.hls_proxy.segment_fetch_policy().origin_object_wait_timeout(); + tokio::time::timeout_at(deadline, async { + loop { + match runtime.prepare(session, HlsOriginAccountWork::Media, class).await { + Ok(_) => return Ok(()), + Err(error) => { + if class != HlsOriginWorkClass::Background { + return Err(error); + } + let now_ms = tuliprox_core::utils::current_time_millis(); + let retry_at = session.read().await.origin_account_rebind.next_rebind_allowed_at_ms; + let Some(wait_ms) = + retry_at.and_then(|retry_at| retry_at.checked_sub(now_ms)).filter(|wait| *wait > 0) + else { + return Err(error); + }; + tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await; + } + } + } + }) + .await + .map_err(|_| HlsBoundAccountAcquireErrorKind::WaitTimedOut)? +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{HlsCtx, HlsSession, HlsSessionKey}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + use tuliprox_core::model::{Config, PlaybackKind}; + use tuliprox_session::ConnectionKind; + + #[derive(Clone, Copy, Debug)] + enum PrepareBehavior { + RetryOnce, + Stall, + } + + #[derive(Debug)] + struct TestRuntime { + attempts: AtomicUsize, + behavior: PrepareBehavior, + } + + impl HlsOriginAccountRuntime for TestRuntime { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + _work: HlsOriginAccountWork, + _class: HlsOriginWorkClass, + ) -> Pin> + Send + 'a>> + { + Box::pin(async move { + let attempt = self.attempts.fetch_add(1, Ordering::AcqRel); + match self.behavior { + PrepareBehavior::Stall => std::future::pending().await, + PrepareBehavior::RetryOnce if attempt == 0 => { + session.write().await.origin_account_rebind.mark_attempt_started( + Arc::from("unavailable"), + tuliprox_core::utils::current_time_millis(), + ); + Err(HlsBoundAccountAcquireErrorKind::Unavailable) + } + PrepareBehavior::RetryOnce => Ok(HlsPreparedOriginAccount { + entry: LiveHlsOriginEntry::parse("https://origin.example/index.m3u8").unwrap(), + snapshot: HlsOriginAccountSnapshot::from_session(&*session.read().await), + }), + } + }) + } + } + + fn fixture(behavior: PrepareBehavior) -> (HlsOriginIoContext, HlsSessionHandle, Arc) { + let runtime = Arc::new(TestRuntime { attempts: AtomicUsize::new(0), behavior }); + let io = HlsOriginIoContext { + ctx: HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:12345".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: Some(runtime.clone()), + }; + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "runtime-backoff"), b"secret", 0))); + (io, session, runtime) + } + + #[tokio::test(start_paused = true)] + async fn background_prepare_waits_for_rebind_backoff_before_retrying() { + let (io, session, runtime) = fixture(PrepareBehavior::RetryOnce); + let mut prepare = Box::pin(prepare_media_origin(Some(&io), &session)); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + tokio::time::advance(std::time::Duration::from_secs(1)).await; + assert!(futures::poll!(prepare.as_mut()).is_pending()); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + tokio::time::advance(std::time::Duration::from_secs(2)).await; + prepare.await.unwrap(); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 2); + } + + #[tokio::test(start_paused = true)] + async fn stalled_background_prepare_is_bounded_by_the_existing_object_budget() { + let (io, session, runtime) = fixture(PrepareBehavior::Stall); + let budget = io.ctx.hls_proxy.segment_fetch_policy().origin_object_wait_timeout(); + let mut prepare = Box::pin(prepare_media_origin(Some(&io), &session)); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + tokio::time::advance(budget).await; + assert_eq!(prepare.await, Err(HlsBoundAccountAcquireErrorKind::WaitTimedOut)); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + } + + #[tokio::test(start_paused = true)] + async fn interactive_prepare_returns_temporary_failure_without_background_waiting() { + let (io, session, runtime) = fixture(PrepareBehavior::RetryOnce); + let io = io.with_grace(true); + assert_eq!(prepare_media_origin(Some(&io), &session).await, Err(HlsBoundAccountAcquireErrorKind::Unavailable)); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + assert!(session.read().await.origin_account_rebind.next_rebind_allowed_at_ms.is_some()); + } +} diff --git a/backend/hls/src/origin_attempt.rs b/backend/hls/src/origin_attempt.rs new file mode 100644 index 000000000..64cca1657 --- /dev/null +++ b/backend/hls/src/origin_attempt.rs @@ -0,0 +1,360 @@ +use crate::{ + begin_hls_origin_account_io_bounded, finish_hls_origin_account_io, origin_account_runtime::prepare_media_origin, + HlsBoundAccountAcquireErrorKind, HlsOriginAccountBinding, HlsOriginAccountIoLeaseGuard, HlsOriginIoContext, + HlsSession, HlsSessionHandle, +}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tuliprox_core::utils::current_time_millis; + +/// Whether the completed media attempt still belongs to the current account. +pub(crate) struct HlsMediaOriginWorkFinish { + pub(crate) generation_valid: bool, +} + +/// Owns the work counter and provider lease for one generation-bound media attempt. +pub(crate) struct HlsMediaOriginAttemptGuard { + generation: u64, + binding: Option, + active_origin_work_count: Option>, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +} + +impl HlsMediaOriginAttemptGuard { + fn matches_session(&self, session: &HlsSession) -> bool { + self.generation == session.activity.origin_work_generation + && match (&self.binding, &session.origin_account_binding) { + (Some(expected), Some(current)) => { + let expected_tag = self + .provider_lease + .as_ref() + .map_or(expected.provider_binding_tag, |(_, lease)| lease.binding().provider_binding_tag); + expected.input_name == current.input_name + && expected.account_name == current.account_name + && expected.session_owner == current.session_owner + && expected.generation == current.generation + && expected.is_active() == current.is_active() + // Reacquiring an expired provider lease may renew its tag without an account handoff. + && expected_tag.is_none_or(|tag| current.provider_binding_tag == Some(tag)) + } + (None, None) => true, + (Some(_), None) | (None, Some(_)) => false, + } + } + + pub(crate) async fn finish(mut self, session: &HlsSessionHandle) -> HlsMediaOriginWorkFinish { + let (generation_valid, refresh_reservation) = { + let mut session = session.write().await; + let generation_valid = self.matches_session(&session); + if self.active_origin_work_count.take().is_some() { + session.finish_origin_work(self.generation); + } + (generation_valid, generation_valid && session.should_refresh_origin_reservation(current_time_millis())) + }; + if let Some((origin_io, lease)) = self.provider_lease.take() { + finish_hls_origin_account_io(&origin_io, session, lease, refresh_reservation).await; + } + HlsMediaOriginWorkFinish { generation_valid } + } +} + +impl Drop for HlsMediaOriginAttemptGuard { + fn drop(&mut self) { + if let Some(count) = self.active_origin_work_count.take() { + count.fetch_sub(1, Ordering::AcqRel); + } + } +} + +/// Captures work and binding atomically, then rejects a handoff during provider acquisition. +pub(crate) async fn prepare_media_origin_attempt( + origin_io: Option<&HlsOriginIoContext>, + session: &HlsSessionHandle, + expected_generation: u64, + acquire_timeout: Duration, +) -> Result { + prepare_media_origin(origin_io, session).await?; + let mut attempt = { + let mut session = session.write().await; + if session.activity.origin_work_generation != expected_generation { + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + let binding = session.origin_account_binding.clone(); + if origin_io.is_some() && binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached) { + return Err(HlsBoundAccountAcquireErrorKind::Detached); + } + let active_origin_work_count = origin_io.map(|_| { + session.start_origin_work(); + Arc::clone(&session.activity.active_origin_work_count) + }); + HlsMediaOriginAttemptGuard { + generation: expected_generation, + binding, + active_origin_work_count, + provider_lease: None, + } + }; + let (Some(origin_io), Some(binding)) = (origin_io, attempt.binding.as_ref()) else { + return Ok(attempt); + }; + let acquired = begin_hls_origin_account_io_bounded(origin_io, session, binding, acquire_timeout).await; + let error = match acquired { + Ok(lease) => { + attempt.provider_lease = Some((origin_io.clone(), lease)); + None + } + Err(error) => Some(error), + }; + let still_current = attempt.matches_session(&*session.read().await); + if !still_current { + attempt.finish(session).await; + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + if let Some(error) = error { + attempt.finish(session).await; + if error.allows_rebind() { + let _ = prepare_media_origin(Some(origin_io), session).await; + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + return Err(error); + } + Ok(attempt) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::prepare_media_origin_attempt; + use crate::{ + HlsBoundAccountAcquireErrorKind, HlsCtx, HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginIoContext, + HlsSession, HlsSessionHandle, HlsSessionKey, + }; + use shared::model::{ConfigInputAliasDto, ConfigInputDto}; + use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, + }; + use tuliprox_core::model::{Config, ConfigInput, PlaybackKind, SourcesConfig}; + use tuliprox_session::ConnectionKind; + + pub(crate) async fn bound_origin_io(session: &HlsSessionHandle) -> (HlsOriginIoContext, HlsOriginAccountBinding) { + let ctx = HlsCtx::for_test(Config::default()); + let input = ConfigInput::from(&ConfigInputDto { + name: Arc::from("account-a"), + url: "http://origin.example/account-a/index.m3u8".to_string(), + enabled: true, + max_connections: 2, + aliases: Some(vec![ConfigInputAliasDto { + name: Arc::from("account-b"), + url: "http://origin.example/account-b/index.m3u8".to_string(), + enabled: true, + max_connections: 2, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }); + ctx.app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(input)], ..SourcesConfig::default() })); + ctx.active_provider.update_config(&ctx.app_config); + let replacement = { + let mut session = session.write().await; + let binding = HlsOriginAccountBinding::new( + Arc::from("account-a"), + Arc::from("account-a"), + &session.proxy_session_id, + 1_000, + ); + let replacement = HlsOriginAccountBinding::rebound( + Arc::clone(&binding.input_name), + Arc::from("account-b"), + binding.session_owner.clone(), + binding.generation + 1, + 2_000, + ); + session.replace_origin_account_binding(Some(binding)); + replacement + }; + let io = HlsOriginIoContext { + ctx, + client_addr: "127.0.0.1:12345".parse().expect("test address"), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: None, + }; + (io, replacement) + } + + #[tokio::test] + async fn handoff_after_lease_acquisition_releases_the_old_attempt_without_touching_its_successor() { + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "lease-handoff"), b"secret", 0))); + let (io, replacement) = bound_origin_io(&session).await; + let original_lease = { + let mut session = session.write().await; + let lease = Arc::new(HlsOriginAccountIoLease::active_for_test( + session.origin_account_binding.as_ref().expect("original binding"), + 1, + )); + session.origin_account_io_lease = Some(Arc::clone(&lease)); + lease + }; + let replacement_lease = Arc::new(HlsOriginAccountIoLease::active_for_test(&replacement, 1)); + let generation = session.read().await.activity.origin_work_generation; + let read_blocker = session.read().await; + let mut prepare = + Box::pin(prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1))); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + // Queue a reader between the atomic start and the provider-slot write. + let mut between_start_and_lease = Box::pin(session.read()); + assert!(futures::poll!(between_start_and_lease.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + let between_start_and_lease = between_start_and_lease.await; + let mut handoff = Box::pin(session.write()); + assert!(futures::poll!(handoff.as_mut()).is_pending()); + drop(between_start_and_lease); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + { + let mut session = handoff.await; + assert_eq!( + original_lease.active_io_count.load(Ordering::Acquire), + 2, + "old lease was acquired before handoff" + ); + session.handoff_origin_account(replacement); + session.origin_account_io_lease = Some(Arc::clone(&replacement_lease)); + } + + assert!(matches!(prepare.await, Err(HlsBoundAccountAcquireErrorKind::StoreRace))); + let session = session.read().await; + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert_eq!(original_lease.active_io_count.load(Ordering::Acquire), 1); + assert_eq!(replacement_lease.active_io_count.load(Ordering::Acquire), 1); + assert!(session.origin_account_io_lease.as_ref().is_some_and(|lease| Arc::ptr_eq(lease, &replacement_lease))); + assert!(session.origin_account_binding.as_ref().expect("new binding").last_origin_io_at_ms.is_none()); + } + + #[tokio::test] + async fn initial_provider_tag_assignment_keeps_the_media_attempt_current() { + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "initial-tag"), b"secret", 0))); + let (io, _) = bound_origin_io(&session).await; + let generation = session.read().await.activity.origin_work_generation; + let attempt = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("initial binding can acquire its first provider lease"); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 1); + assert!(attempt.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert!(session.read().await.origin_account_io_lease.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn same_account_reacquisition_after_provider_lease_expiry_keeps_the_media_attempt_current() { + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "lease-expiry"), b"secret", 0))); + let (io, _) = bound_origin_io(&session).await; + let generation = session.read().await.activity.origin_work_generation; + let first = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("first lease acquisition"); + assert!(first.finish(&session).await.generation_valid); + let original = session.read().await.origin_account_binding.clone().expect("original binding"); + let original_tag = original.provider_binding_tag.expect("first lease has a provider tag"); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + + tokio::time::advance(Duration::from_secs(tuliprox_session::provider_leases::STARTING_LEASE_TTL_SECS + 1)).await; + + let reacquired = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("same account reacquires its expired provider lease"); + { + let session = session.read().await; + let current = session.origin_account_binding.as_ref().expect("reacquired binding"); + assert_eq!( + (¤t.input_name, ¤t.account_name, ¤t.session_owner, current.generation), + (&original.input_name, &original.account_name, &original.session_owner, original.generation) + ); + assert_eq!(session.activity.origin_work_generation, generation); + assert_ne!(current.provider_binding_tag.expect("new lease has a provider tag"), original_tag); + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 1); + } + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert!(reacquired.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert!(session.read().await.origin_account_io_lease.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn concurrent_media_attempts_join_the_reacquired_provider_lease_with_its_current_tag() { + let session = Arc::new(tokio::sync::RwLock::new(HlsSession::new( + HlsSessionKey::new(1, "concurrent-lease-expiry"), + b"secret", + 0, + ))); + let (io, _) = bound_origin_io(&session).await; + let generation = session.read().await.activity.origin_work_generation; + let baseline = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("baseline provider lease"); + assert!(baseline.finish(&session).await.generation_valid); + let original_tag = session.read().await.origin_account_binding.as_ref().expect("binding").provider_binding_tag; + tokio::time::advance(Duration::from_secs(tuliprox_session::provider_leases::STARTING_LEASE_TTL_SECS + 1)).await; + + // Queue both snapshots before either attempt can replace the expired provider tag. + let read_blocker = session.read().await; + let mut first = Box::pin(prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1))); + assert!(futures::poll!(first.as_mut()).is_pending()); + let mut second = + Box::pin(prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1))); + assert!(futures::poll!(second.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(first.as_mut()).is_pending()); + assert!(futures::poll!(second.as_mut()).is_pending()); + // Let the first reserve its slot and the second wait for that slot's acquisition. + assert!(futures::poll!(first.as_mut()).is_pending()); + assert!(futures::poll!(second.as_mut()).is_pending()); + let first = first.await.expect("first attempt reacquires the provider lease"); + let second = second.await.expect("second attempt joins the reacquired provider lease"); + { + let session = session.read().await; + let current = session.origin_account_binding.as_ref().expect("binding"); + let joined = second.provider_lease.as_ref().expect("joined provider lease").1.binding(); + assert_ne!(current.provider_binding_tag, original_tag); + assert_eq!(joined.provider_binding_tag, current.provider_binding_tag); + assert_eq!(joined.playback_request_id, current.playback_request_id); + assert_eq!(session.activity.origin_work_generation, generation); + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 2); + assert_eq!( + session + .origin_account_io_lease + .as_ref() + .expect("shared I/O lease") + .active_io_count + .load(Ordering::Acquire), + 2 + ); + } + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert!(first.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert!(second.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert!(session.read().await.origin_account_io_lease.is_none()); + } +} diff --git a/backend/hls/src/refresh/commit.rs b/backend/hls/src/refresh/commit.rs index c9f8d117a..8b3c60d5e 100644 --- a/backend/hls/src/refresh/commit.rs +++ b/backend/hls/src/refresh/commit.rs @@ -134,6 +134,18 @@ pub(super) fn commit_fetched_manifest_with_acceptance_mode( staged_switch: Option<&HlsStagedSwitchCommit>, ) -> Result<(HlsManifestCommitProgressEvidence, bool, bool), HlsManifestCommitError> { ensure_refresh_origin_work_generation_is_current(session, request)?; + if !session + .origin_control + .qualification + .pending_is_current(&crate::short_manifest::HlsOriginQualificationBasis::capture(session)) + || (matches!( + session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Checking { .. } + ) && session.origin_control.qualification.generation() + != Some(session.origin_control.acceptance_generation)) + { + return Err(switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated)); + } let existing_transient_reason = match &session.mode { HlsSessionMode::TransientPassthrough { reason } => Some(reason.clone()), HlsSessionMode::NormalCacheTimeline => None, @@ -178,6 +190,7 @@ pub(super) fn commit_fetched_manifest_with_acceptance_mode( if result.is_ok() { if matches!(session.mode, HlsSessionMode::TransientPassthrough { .. }) { session.startup = None; + session.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Unchecked; } clear_satisfied_fresh_manifest_requirement(session, request); } @@ -254,6 +267,8 @@ fn commit_normal_fetched_manifest( )?; update_origin_provider_session_headers(session, fetched); mark_manifest_acceptance_success(session, fetched, &quality, fetch_finished_at_ms); + super::short_manifest::record_origin_qualification(session, manifest.segments.len()); + queue_and_render_normal_manifest(session, request, fetch_finished_at_ms); Ok((HlsManifestCommitProgressEvidence::CacheTimeline(refresh_timing), true, true)) } @@ -431,9 +446,23 @@ fn evaluate_manifest_acceptance_for_commit( match evaluate_manifest_acceptance(session, fetched, timeline, &fetch_context, now_ms, mode) { HlsManifestAcceptanceDecision::Accept { quality } | HlsManifestAcceptanceDecision::AcceptHostSwitch { quality } => Ok(quality), - HlsManifestAcceptanceDecision::RetryCurrentTarget { .. } => Err(HlsManifestCommitError::RetryCurrentTarget), + HlsManifestAcceptanceDecision::RetryCurrentTarget { .. } => { + if matches!( + session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Qualified { .. } + ) { + session.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Unchecked; + } + Err(HlsManifestCommitError::RetryCurrentTarget) + } HlsManifestAcceptanceDecision::Reject { reason, .. } => { session.origin_control.path_condition = crate::origin_progress::HlsOriginPathCondition::AcceptanceConflict; + if matches!( + session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Qualified { .. } + ) { + session.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Unchecked; + } let log_reason = HlsManifestRejectLogReason::from(reason.clone()); warn!( "HLS origin manifest rejected: session={} proxy_session={} reason={} media_sequence={} segments={}", @@ -502,6 +531,7 @@ fn mark_manifest_acceptance_success( quality: &HlsManifestOriginQuality, observed_at_ms: u64, ) { + session.origin_account_manifest_pending = None; let binding = Url::parse(&fetched.resolved_request_url) .ok() .and_then(|request_url| HlsManifestOriginBinding::new(request_url, fetched.provider_url_index).ok()); @@ -570,6 +600,10 @@ fn prepare_transient_manifest_rewrite( let handoff_discontinuity_sequence = session .pending_handoff_discontinuity_sequence .or_else(|| input.quality.requires_handoff_discontinuity.then_some(0)); + let rewrite_options = TransientRewriteOptions { + handoff_discontinuity_sequence, + origin_account_generation: session.transient.origin_account_generation(), + }; let append_context = (handoff_discontinuity_sequence.is_none() && matches!(input.manifest_semantics.playlist_type, Some(HlsPlaylistType::Event))) .then(|| { @@ -606,16 +640,17 @@ fn prepare_transient_manifest_rewrite( incremental_rewrite = true; appended } else { - TransientManifestRewriter::rewrite( + TransientManifestRewriter::rewrite_with_options( input.body, input.final_manifest_url, &session.proxy_session_id, input.reverse_proxy_rewrite_secret, input.rendered_at_ms, input.transient_resource_ttl_ms, + rewrite_options, ) } - } else if handoff_discontinuity_sequence.is_some() { + } else { TransientManifestRewriter::rewrite_with_options( input.body, input.final_manifest_url, @@ -623,16 +658,7 @@ fn prepare_transient_manifest_rewrite( input.reverse_proxy_rewrite_secret, input.rendered_at_ms, input.transient_resource_ttl_ms, - TransientRewriteOptions { handoff_discontinuity_sequence }, - ) - } else { - TransientManifestRewriter::rewrite( - input.body, - input.final_manifest_url, - &session.proxy_session_id, - input.reverse_proxy_rewrite_secret, - input.rendered_at_ms, - input.transient_resource_ttl_ms, + rewrite_options, ) }; let next_transient_discontinuity_sequence = if handoff_discontinuity_sequence.is_some() { @@ -816,6 +842,7 @@ fn commit_normal_manifest( &request.reverse_proxy_rewrite_secret, rendered_at_ms, request.transient_resource_ttl_ms, + session.transient.origin_account_generation(), ); let manifest = &manifest; let previous_highwater = session.origin_seq_highwater; @@ -871,8 +898,6 @@ fn commit_normal_manifest( if provisioning_handoff { limit_publishable_normal_provisioning_handoff_tail(session, &request.strip, manifest.segments.len()); } - queue_and_render_normal_manifest(session, request, rendered_at_ms); - let last_segment_duration_ms = manifest.segments.last().map(|segment| segment.duration_ms); let target_duration_ms = manifest.target_duration.map(|duration| u64::from(duration) * 1_000); let progress = if staged_switch.is_some() { @@ -983,11 +1008,12 @@ pub(super) fn materialize_normal_key_resources( reverse_proxy_rewrite_secret: &[u8], now_ms: u64, ttl_ms: u64, + origin_account_generation: u64, ) -> Vec { let mut resources = Vec::new(); for encryption in manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { let extension = key_resource_extension(&encryption.resolved_origin_uri); - let resource = TransientResourceRef::new( + let mut resource = TransientResourceRef::new( TransientResourceKind::Key, encryption.resolved_origin_uri.clone(), reverse_proxy_rewrite_secret, @@ -995,6 +1021,7 @@ pub(super) fn materialize_normal_key_resources( ttl_ms, Some(extension.clone()), ); + resource.scope_to_origin_account(origin_account_generation, reverse_proxy_rewrite_secret); encryption.proxy_resource_id = Some(resource.id.0.clone()); encryption.proxy_resource_extension = Some(extension); resources.push(resource); diff --git a/backend/hls/src/refresh/failure.rs b/backend/hls/src/refresh/failure.rs index ff617df49..90a396486 100644 --- a/backend/hls/src/refresh/failure.rs +++ b/backend/hls/src/refresh/failure.rs @@ -163,7 +163,7 @@ pub(super) fn apply_manifest_fetch_failure_signal( } else if signal.is_hard() { Some(HlsPostRefreshAvailabilityReason::HardManifestFailure) } else { - None + Some(HlsPostRefreshAvailabilityReason::RetryableManifestFailure) }; reason.map_or(HlsPostRefreshAvailabilityAction::None, |reason| HlsPostRefreshAvailabilityAction::Reevaluate { reason, diff --git a/backend/hls/src/refresh/mod.rs b/backend/hls/src/refresh/mod.rs index 17c5c0e7b..c76c3a2ff 100644 --- a/backend/hls/src/refresh/mod.rs +++ b/backend/hls/src/refresh/mod.rs @@ -50,6 +50,7 @@ use tuliprox_core::model::{AppConfig, HlsManifestRecoveryBurstConfig, ReversePro mod commit; mod failure; +mod short_manifest; mod switch_staging; mod timing; @@ -84,6 +85,8 @@ pub struct OriginRefreshState { pub last_success_at_ms: Option, pub last_error_at_ms: Option, pub in_flight: bool, + /// Ownership token for the currently running refresh, independent of work generations and timestamps. + pub refresh_identity: Option>, } impl OriginRefreshState { @@ -92,6 +95,7 @@ impl OriginRefreshState { pub fn mark_started(&mut self, now_ms: u64) { self.last_fetch_started_at_ms = Some(now_ms); self.in_flight = true; + self.refresh_identity = Some(Arc::new(())); } fn mark_success_with_timing( @@ -115,6 +119,7 @@ impl OriginRefreshState { self.last_error_at_ms = None; self.consecutive_failures = 0; self.in_flight = false; + self.refresh_identity = None; self.next_fetch_allowed_at_ms = fetch_started_at_ms.saturating_add(refresh_interval_ms); refresh_interval_ms } @@ -125,12 +130,14 @@ impl OriginRefreshState { let next_retry_delay_ms = self.next_failure_backoff_ms(); self.consecutive_failures = self.consecutive_failures.saturating_add(1); self.in_flight = false; + self.refresh_identity = None; self.next_fetch_allowed_at_ms = failed_at_ms.saturating_add(next_retry_delay_ms); } fn mark_invalidated(&mut self, finished_at_ms: u64) { self.last_fetch_finished_at_ms = Some(finished_at_ms); self.in_flight = false; + self.refresh_identity = None; } fn next_failure_backoff_ms(&self) -> u64 { @@ -167,6 +174,8 @@ pub struct OriginRefreshRequest { pub transient_resource_ttl_ms: u64, pub manifest_commit_requirement: HlsManifestCommitRequirement, pub fresh_manifest_requirement_generation: Option, + /// Filled at refresh start; a completion may only change the state owned by this token. + pub refresh_identity: Option>, pub acceptance_directive: HlsManifestAcceptanceDirective, pub access_lease_id: Option, pub now_ms: u64, @@ -183,6 +192,7 @@ pub struct HlsPostRefreshRuntime { pub enum HlsPostRefreshAvailabilityReason { DeterministicTimelineConflict, HardManifestFailure, + RetryableManifestFailure, } impl HlsPostRefreshAvailabilityReason { @@ -190,6 +200,7 @@ impl HlsPostRefreshAvailabilityReason { match self { Self::DeterministicTimelineConflict => "deterministic_timeline_conflict", Self::HardManifestFailure => "hard_manifest_failure", + Self::RetryableManifestFailure => "retryable_manifest_failure", } } } @@ -326,6 +337,34 @@ async fn mark_origin_refresh_started_with_outcome( request: &mut OriginRefreshRequest, fetch_started_at_ms: u64, ) -> HlsOriginRefreshTriggerOutcome { + if let Some(io) = request.origin_io.as_ref() { + let needs_prepare = { + let session = request.session.read().await; + io.prepared_account.as_ref().is_none_or(|snapshot| !snapshot.matches_session(&session)) + || session.origin_account_binding.as_ref().is_some_and(|binding| { + binding.is_detached() + || crate::origin::hls_origin_account_status(&io.ctx, binding) + != crate::origin::HlsOriginAccountStatus::Known + }) + }; + if let Some(runtime) = io.account_runtime.as_ref().filter(|_| needs_prepare) { + // A replacement result must not inherit an allocation prepared for the old entry. + drop(io.take_preacquired_provider_handle()); + let class = + if io.allow_grace { HlsOriginWorkClass::ManifestInteractive } else { HlsOriginWorkClass::Background }; + let Ok(prepared) = runtime + .prepare(&request.session, crate::origin_account_runtime::HlsOriginAccountWork::Manifest, class) + .await + else { + return HlsOriginRefreshTriggerOutcome::SessionUnavailable; + }; + request.origin_provider_session_headers = prepared.snapshot.provider_session_headers.clone(); + if let Some(io) = request.origin_io.as_mut() { + io.prepared_account = Some(prepared.snapshot); + } + request.origin_entry = prepared.entry; + } + } let metrics = Arc::clone(request.segment_worker_pool.metrics()); if let Some(guard) = request.acceptance_directive.recovery_pressure_guard.clone() { let hls_proxy = Arc::clone(&request.hls_proxy); @@ -358,7 +397,13 @@ fn mark_origin_refresh_started_in_session( fetch_started_at_ms: u64, metrics: &super::HlsCacheMetrics, ) -> HlsOriginRefreshTriggerOutcome { - if session.is_gc_marked_for_removal() { + if session.is_gc_marked_for_removal() + || request + .origin_io + .as_ref() + .and_then(|io| io.prepared_account.as_ref()) + .is_some_and(|snapshot| !snapshot.matches_session(session)) + { return HlsOriginRefreshTriggerOutcome::SessionUnavailable; } if session.origin_refresh.in_flight { @@ -381,7 +426,8 @@ fn mark_origin_refresh_started_in_session( return HlsOriginRefreshTriggerOutcome::InFlight; } if fetch_started_at_ms < session.origin_refresh.next_fetch_allowed_at_ms - && request.manifest_commit_requirement.fresh_reason().is_none() + && (request.manifest_commit_requirement.fresh_reason().is_none() + || session.origin_refresh.consecutive_failures > 0) { metrics.record_refresh_skipped(); let wait_ms = session.origin_refresh.next_fetch_allowed_at_ms.saturating_sub(fetch_started_at_ms); @@ -395,6 +441,7 @@ fn mark_origin_refresh_started_in_session( retry_at_ms: session.origin_refresh.next_fetch_allowed_at_ms, }; } + short_manifest::prepare_qualification_refresh(request, session, fetch_started_at_ms); if let Some(origin_io) = request.origin_io.as_mut() { origin_io.started_generation = Some(session.start_origin_work()); } @@ -403,6 +450,7 @@ fn mark_origin_refresh_started_in_session( .fresh_reason() .and_then(|reason| session.fresh_manifest_commit_requirement_generation(reason)); session.origin_refresh.mark_started(fetch_started_at_ms); + request.refresh_identity = session.origin_refresh.refresh_identity.clone(); metrics.record_refresh_started(); info!( "HLS origin manifest refresh started: session={} proxy_session={}", @@ -496,7 +544,26 @@ async fn refresh_and_commit(mut request: OriginRefreshRequest, fetch_started_at_ committed_evidence_session, ) = { let mut session = request.session.write().await; + let owns_refresh = match (&request.refresh_identity, &session.origin_refresh.refresh_identity) { + (Some(expected), Some(current)) => Arc::ptr_eq(expected, current), + (None, _) => true, + (Some(_), None) => false, + }; match result { + // A handoff can reset refresh state and start a successor in the same millisecond. + Ok(_) | Err(_) if !owns_refresh => { + metrics.record_refresh_skipped(); + (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) + } + // Invalidated work must release its own singleflight flag even when no successor was published. + Ok(_) | Err(_) + if !origin_work_state.generation_valid + || !refresh_origin_work_generation_is_current(&session, &request) => + { + session.origin_refresh.mark_invalidated(fetch_finished_at_ms); + metrics.record_refresh_skipped(); + (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) + } Ok(CommittedOriginManifest { fetched, progress_evidence, wake_segment_scheduler, wake_map_scheduler }) => { let pending_manifest_follow_up = Some((session.proxy_session_id.clone(), session.target_duration)); let prepared_terminal_target_duration_ms = if early_prepared_terminal_target_duration_ms.is_none() { @@ -559,21 +626,8 @@ async fn refresh_and_commit(mut request: OriginRefreshRequest, fetch_started_at_ record_superseded_manifest_recovery_completion(&mut session, metrics.as_ref(), fetch_finished_at_ms); (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) } - Err(err) - if !origin_work_state.generation_valid - || !refresh_origin_work_generation_is_current(&session, &request) => - { - session.origin_refresh.mark_invalidated(fetch_finished_at_ms); - metrics.record_refresh_skipped(); - debug!( - "HLS origin manifest refresh completion discarded: session={} proxy_session={} reason=origin-work-generation-invalidated error={}", - safe_session_key(&session.key), - safe_proxy_session_id(&session.proxy_session_id), - err.log_label() - ); - (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) - } Err(err) => { + session.origin_control.qualification.record_failure(); session.origin_refresh.mark_failure(fetch_finished_at_ms); metrics.record_refresh_failed(); let post_refresh_action = apply_manifest_fetch_failure_signal(&mut session, &err, fetch_finished_at_ms); @@ -864,6 +918,7 @@ fn manifest_fetch_context(request: &OriginRefreshRequest) -> HlsOriginManifestFe HlsOriginManifestFetchContext { app_config: Arc::clone(&request.app_config), session: Arc::clone(&request.session), + availability_runtime: request.post_refresh_runtime.as_ref().map(|runtime| runtime.ctx.clone()), origin_entry: request.origin_entry.clone(), headers: hls_origin_headers_with_provider_session(&request.headers, &request.origin_provider_session_headers), client: request.client.clone(), @@ -880,7 +935,7 @@ fn manifest_fetch_context(request: &OriginRefreshRequest) -> HlsOriginManifestFe async fn fetch_and_commit_manifest_with_policy( request: &mut OriginRefreshRequest, ) -> Result { - let (established_recovery_binding, prefetch_trigger) = { + let (mut established_recovery_binding, prefetch_trigger) = { let session = request.session.read().await; if !refresh_origin_work_generation_is_current(&session, request) { return Err(OriginManifestFetchError::RetryExhausted); @@ -898,6 +953,9 @@ async fn fetch_and_commit_manifest_with_policy( (session.established_manifest_recovery_binding(), trigger) }; let fetch_context = manifest_fetch_context(request); + if let Some(result) = short_manifest::resume_qualification(request, &fetch_context).await { + return result; + } let mut recovery_suppression_logged = false; if prefetch_trigger.starts_episode() { if let Some(binding) = established_recovery_binding.clone() { @@ -944,6 +1002,20 @@ async fn fetch_and_commit_manifest_with_policy( } Err(initial_error) => return Err(initial_error), }; + // A handoff first resolves the new entry. Recovery may use only that concrete target, + // while acceptance still compares against the retained timeline and pinned host. + { + let mut session = request.session.write().await; + if session.origin_account_manifest_pending.is_some() { + if !refresh_origin_work_generation_is_current(&session, request) { + return Err(OriginManifestFetchError::RetryExhausted); + } + session.origin_control.manifest_origin_binding = url::Url::parse(&fetched.resolved_request_url) + .ok() + .and_then(|url| HlsManifestOriginBinding::new(url, fetched.provider_url_index).ok()); + established_recovery_binding = session.established_manifest_recovery_binding(); + } + } Box::pin(commit_initial_fetched_manifest( request, &fetch_context, @@ -1057,6 +1129,9 @@ async fn commit_initial_fetched_manifest( established_recovery_binding: Option, recovery_suppression_logged: bool, ) -> Result { + if let Some(result) = short_manifest::qualify_short_manifest_if_needed(request, fetch_context, &fetched).await { + return result; + } let acceptance_mode = request.manifest_commit_requirement.acceptance_mode(); let selected_report = score_hls_manifest_candidate_for_selection_log(fetch_context, &fetched, acceptance_mode).await; diff --git a/backend/hls/src/refresh/short_manifest.rs b/backend/hls/src/refresh/short_manifest.rs new file mode 100644 index 000000000..51a97738f --- /dev/null +++ b/backend/hls/src/refresh/short_manifest.rs @@ -0,0 +1,215 @@ +//! Startup qualification shares recovery selection, singleflight and retry receipts. + +use super::{commit_manifest_recovery_candidate, CommittedOriginManifest, OriginRefreshRequest}; +use crate::{ + hls_origin_log_value, + manifest_acceptance::HlsManifestAcceptanceTrigger, + manifest_fetch::{ + retry_hls_origin_manifest_recovery_chain, FetchedOriginManifest, HlsOriginManifestFetchContext, + OriginManifestFetchError, + }, + manifest_origin_binding::HlsManifestOriginBinding, + safe_proxy_session_id, safe_session_key, + short_manifest::{HlsOriginQualification, HlsOriginQualificationBasis, HlsQualifiedOriginWindows}, + HlsFreshManifestRequiredReason, HlsSession, +}; +use tuliprox_parser::hls::origin_manifest::{ + parse_manifest_semantics, parse_origin_media_manifest, OriginManifestParseOutcome, +}; +use url::Url; + +pub(super) fn prepare_qualification_refresh(request: &mut OriginRefreshRequest, session: &mut HlsSession, now_ms: u64) { + let basis = HlsOriginQualificationBasis::capture(session); + if !session.origin_control.qualification.pending_is_current(&basis) { + session.origin_control.qualification = HlsOriginQualification::Unchecked; + } + if !session.origin_is_qualified() { + return; + } + let Some(reason) = request.manifest_commit_requirement.fresh_reason() else { + return; + }; + let pending_requirement = session.fresh_manifest_commit_requirement_generation(reason).is_some(); + let already_qualified = !pending_requirement + && match reason { + HlsFreshManifestRequiredReason::ColdStart + | HlsFreshManifestRequiredReason::PreviousHardManifestFailure + | HlsFreshManifestRequiredReason::StaleEntryWindow => true, + HlsFreshManifestRequiredReason::ProvisioningHandoff => { + session.pending_handoff_discontinuity_sequence.is_none() + } + HlsFreshManifestRequiredReason::ExpiredRevalidation => { + // Each qualification may still be waiting for its media, + // including revalidation of an already published session. + session.origin_qualification_awaits_publication() + || crate::manifest_commit::has_servable_committed_manifest(session, now_ms) + } + }; + if already_qualified { + // Another lease completed the foundation check. Freshness remains a + // normal refresh requirement, without spending another configured burst. + request.manifest_commit_requirement = super::HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::StaleEntryWindow, + }; + } else if matches!( + reason, + HlsFreshManifestRequiredReason::ExpiredRevalidation + | HlsFreshManifestRequiredReason::PreviousHardManifestFailure + | HlsFreshManifestRequiredReason::ProvisioningHandoff + ) { + session.origin_control.qualification = HlsOriginQualification::Unchecked; + } +} + +pub(super) async fn qualify_short_manifest_if_needed( + request: &OriginRefreshRequest, + context: &HlsOriginManifestFetchContext, + fetched: &FetchedOriginManifest, +) -> Option> { + if parse_manifest_semantics(&fetched.body).window_policy().preserves_full_manifest() { + return None; + } + let OriginManifestParseOutcome::Normal(manifest) = + parse_origin_media_manifest(&fetched.body, &fetched.final_manifest_url) + else { + return None; + }; + if !(1..3).contains(&manifest.segments.len()) { + return None; + } + let binding = Url::parse(&fetched.resolved_request_url) + .ok() + .and_then(|url| HlsManifestOriginBinding::new(url, fetched.provider_url_index).ok())?; + let qualified = { + let session = request.session.read().await; + session.origin_is_qualified() + && session.origin_control.manifest_origin_binding.as_ref() == Some(&binding) + && crate::manifest_fetch::fetched_effective_manifest_host(fetched).as_deref() + == session.origin_control.pinned_host.as_deref() + }; + if qualified { + return None; + } + let mut qualification_context = context.clone(); + if !fetched.provider_session_headers.is_empty() { + let mut session = request.session.write().await; + if !super::commit::refresh_origin_work_generation_is_current(&session, request) { + return Some(Err(OriginManifestFetchError::RecoveryUnavailable { + reason: crate::manifest_fetch::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + })); + } + // HTTP session cookies belong to the transport. Keeping this successful + // response's cookie does not commit its media or qualify its short window. + session.origin_provider_session_headers = crate::headers::merge_hls_provider_session_headers( + &session.origin_provider_session_headers, + &fetched.provider_session_headers, + ); + qualification_context.headers = + crate::hls_origin_headers_with_provider_session(&context.headers, &session.origin_provider_session_headers); + } + let result = run_qualification(request, &qualification_context, binding).await.map(|mut committed| { + committed.fetched.attempts = committed.fetched.attempts.saturating_add(fetched.attempts.saturating_sub(1)); + committed.fetched.candidate_requests = + committed.fetched.candidate_requests.saturating_add(fetched.candidate_requests); + committed + }); + Some(result) +} + +pub(super) async fn resume_qualification( + request: &OriginRefreshRequest, + context: &HlsOriginManifestFetchContext, +) -> Option> { + let target = { + let mut session = request.session.write().await; + let target = session.origin_qualification_retry_target().cloned()?; + if let Some(reason) = request.manifest_commit_requirement.fresh_reason() { + let trigger = request.manifest_commit_requirement.strict_handoff_trigger(); + if trigger.starts_episode() + && session.fresh_manifest_commit_requirement_generation(reason).is_some() + && session + .origin_control + .acceptance_episode + .as_ref() + .is_some_and(|episode| episode.trigger() != trigger) + { + // A new lifecycle finding gets one new full check of the known + // concrete target. Further leases inherit that check's receipt. + session.origin_control.qualification = HlsOriginQualification::Unchecked; + } + } + target + }; + Some(run_qualification(request, context, target).await) +} + +async fn run_qualification( + request: &OriginRefreshRequest, + context: &HlsOriginManifestFetchContext, + binding: HlsManifestOriginBinding, +) -> Result { + let trigger = { + let session = request.session.read().await; + session + .origin_qualification_retry_target() + .and_then(|_| { + session + .origin_control + .acceptance_episode + .as_ref() + .map(crate::manifest_acceptance::HlsManifestAcceptanceEpisode::trigger) + }) + .unwrap_or_else(|| { + let trigger = super::manifest_recovery_trigger(request); + if trigger.starts_episode() { + trigger + } else { + HlsManifestAcceptanceTrigger::Observe + } + }) + }; + Box::pin(retry_hls_origin_manifest_recovery_chain( + context, + binding, + None, + None, + trigger, + request.manifest_commit_requirement.acceptance_mode(), + |fetched, mode| Box::pin(commit_manifest_recovery_candidate(request, fetched, mode)), + )) + .await +} + +pub(super) fn record_origin_qualification(session: &mut HlsSession, segment_count: usize) { + let basis = HlsOriginQualificationBasis::capture(session); + if session.origin_control.qualification.qualified_windows(&basis).is_some() { + return; + } + let generation = session.origin_control.acceptance_generation; + let full_burst = session.origin_control.qualification.generation() == Some(generation) + && matches!(session.origin_control.qualification, HlsOriginQualification::Checking { .. }) + && session + .origin_control + .acceptance_episode + .as_ref() + .is_some_and(|episode| episode.generation == generation && episode.full_burst_completed); + let windows = if (1..3).contains(&segment_count) && full_burst { + log::warn!( + "HLS short origin window qualified: RFC 8216 section 6.2.2 recommends three target durations; host={} session={} proxy_session={} segments={segment_count} qualification={} burst={}", + session.origin_control.pinned_host.as_deref().map_or_else(|| "none".to_string(), hls_origin_log_value), + safe_session_key(&session.key), safe_proxy_session_id(&session.proxy_session_id), generation.0, + session.origin_control.acceptance_episode.as_ref().map_or(0, |episode| episode.completed_burst_candidates) + ); + HlsQualifiedOriginWindows::ShortAllowed + } else if segment_count >= 3 { + HlsQualifiedOriginWindows::Regular + } else { + return; + }; + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis, + generation, + windows, + publication_boundary: session.last_normal_manifest_commit_identity(), + }; +} diff --git a/backend/hls/src/refresh/switch_staging.rs b/backend/hls/src/refresh/switch_staging.rs index 7cc21218e..9c29c0676 100644 --- a/backend/hls/src/refresh/switch_staging.rs +++ b/backend/hls/src/refresh/switch_staging.rs @@ -325,25 +325,26 @@ pub(super) async fn stage_alternative_manifest_switch( return Err(switch_staging_error(HlsManifestRejectLogReason::SwitchResourceUnavailable)); } }; - let candidate_key_resources = materialize_normal_key_resources( - &mut manifest, - &request.reverse_proxy_rewrite_secret, - request.now_ms, - request.transient_resource_ttl_ms, - ); let effective_host = fetched_effective_manifest_host(fetched) .ok_or_else(|| switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated))?; let effective_host_id = effective_origin_host_id(&effective_host); - let (generation, preview) = { + let (generation, preview, candidate_key_resources) = { let session = request.session.read().await; let generation = switch_staging_generation(&session) .ok_or_else(|| switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated))?; if !generation.candidate_identity.matches_candidate(Some(&effective_host), &fetched.body) { return Err(switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated)); } + let candidate_key_resources = materialize_normal_key_resources( + &mut manifest, + &request.reverse_proxy_rewrite_secret, + request.now_ms, + request.transient_resource_ttl_ms, + session.transient.origin_account_generation(), + ); let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).map_err(handoff_preview_error)?; - (generation, preview) + (generation, preview, candidate_key_resources) }; let first_segment = preview .segments diff --git a/backend/hls/src/refresh/tests.rs b/backend/hls/src/refresh/tests.rs index a72e98143..947b13d59 100644 --- a/backend/hls/src/refresh/tests.rs +++ b/backend/hls/src/refresh/tests.rs @@ -61,12 +61,15 @@ use super::{ }; mod acceptance; +mod account_handoff; mod archive_master; mod critical_handoff; mod headers_redirects; mod origin_binding; mod ownership; +mod reduced_retry; mod retry_policy; +mod short_manifest; mod startup_representation; mod support; mod switch_staging; diff --git a/backend/hls/src/refresh/tests/acceptance.rs b/backend/hls/src/refresh/tests/acceptance.rs index 8751977c8..aa25e6355 100644 --- a/backend/hls/src/refresh/tests/acceptance.rs +++ b/backend/hls/src/refresh/tests/acceptance.rs @@ -247,6 +247,7 @@ pub(in crate::refresh::tests) fn install_stored_provisioning_manifest(session: & host_local_index: u32::try_from(*proxy_seq).unwrap_or(u32::MAX), }, proxy_seq: *proxy_seq, + media_start_ms: *proxy_seq * 2_000, duration_ms: 2_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -267,6 +268,7 @@ pub(in crate::refresh::tests) fn install_stored_provisioning_manifest(session: & session.publishable_origin_head_proxy_seq = segment_proxy_seqs.first().copied(); session.publishable_origin_tail_proxy_seq = segment_proxy_seqs.last().copied(); session.proxy_next_seq = Some(3); + session.next_media_start_ms = 6_000; session.target_duration = Some(2); assert_eq!( session.store_rendered_manifest(RenderedManifest { diff --git a/backend/hls/src/refresh/tests/account_handoff.rs b/backend/hls/src/refresh/tests/account_handoff.rs new file mode 100644 index 000000000..52223dd58 --- /dev/null +++ b/backend/hls/src/refresh/tests/account_handoff.rs @@ -0,0 +1,139 @@ +use super::{ + mark_origin_refresh_started_with_outcome, refresh_and_commit, test_origin_refresh_request, test_session, + HlsOriginRefreshTriggerOutcome, +}; +use crate::{HlsCtx, HlsOriginIoContext, HlsSessionHandle}; +use std::sync::Arc; +use tuliprox_core::model::{Config, PlaybackKind}; +use tuliprox_session::ConnectionKind; + +pub(super) fn request_with_origin_work(session: &HlsSessionHandle) -> super::OriginRefreshRequest { + let mut request = test_origin_refresh_request(Arc::clone(session)); + request.origin_io = Some(HlsOriginIoContext { + ctx: HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:1234".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: None, + }); + request +} + +#[tokio::test] +async fn invalidated_refresh_without_successful_handoff_releases_its_singleflight() { + let session = test_session(); + let mut old = request_with_origin_work(&session); + assert_eq!(mark_origin_refresh_started_with_outcome(&mut old, 100).await, HlsOriginRefreshTriggerOutcome::Started); + // A failed account replacement invalidates work but publishes no successor state. + session.write().await.invalidate_queued_origin_work(); + refresh_and_commit(old, 100).await; + { + let current = session.read().await; + assert!(!current.origin_refresh.in_flight); + assert_eq!(current.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), 0); + } + let mut retry = request_with_origin_work(&session); + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut retry, 100).await, + HlsOriginRefreshTriggerOutcome::Started + ); + session.write().await.invalidate_queued_origin_work(); + refresh_and_commit(retry, 100).await; +} + +#[tokio::test] +async fn invalidated_refresh_cannot_release_a_successor_started_in_the_same_millisecond() { + let session = test_session(); + let mut old = request_with_origin_work(&session); + assert_eq!(mark_origin_refresh_started_with_outcome(&mut old, 100).await, HlsOriginRefreshTriggerOutcome::Started); + { + let mut current = session.write().await; + current.invalidate_queued_origin_work(); + current.origin_refresh = super::OriginRefreshState::default(); + } + let mut successor = request_with_origin_work(&session); + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut successor, 100).await, + HlsOriginRefreshTriggerOutcome::Started + ); + let successor_state = session.read().await.origin_refresh.clone(); + refresh_and_commit(old, 100).await; + { + let current = session.read().await; + assert!(current.origin_refresh.in_flight); + assert!(Arc::ptr_eq( + current.origin_refresh.refresh_identity.as_ref().unwrap(), + successor_state.refresh_identity.as_ref().unwrap() + )); + assert_eq!(current.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), 1); + } + session.write().await.invalidate_queued_origin_work(); + refresh_and_commit(successor, 100).await; + assert!(!session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn account_handoff_preserves_cached_ciphertext_key_and_assigns_new_keys_to_uncached_media() { + let session = test_session(); + let request = test_origin_refresh_request(Arc::clone(&session)); + let fetched = super::fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-KEY:METHOD=AES-128,URI=\"shared.key\"\n#EXTINF:4,\n10.ts\n#EXTINF:4,\n11.ts\n#EXTINF:4,\n12.ts\n"); + let mut current = session.write().await; + super::commit_fetched_manifest(&mut current, &fetched, &request, 100).unwrap(); + let old_key = current.segments[&0].encryption.clone().unwrap().resource_id; + current.segments.get_mut(&0).unwrap().status = + crate::SegmentCacheStatus::Ready { content_length: 16, ready_at_ms: 100 }; + let owner = current.proxy_session_id.clone(); + current.handoff_origin_account(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("new"), + owner.0, + 1, + 101, + )); + let next = super::fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:13\n#EXT-X-KEY:METHOD=AES-128,URI=\"shared.key\"\n#EXTINF:4,\n13.ts\n#EXTINF:4,\n14.ts\n#EXTINF:4,\n15.ts\n"); + super::commit_fetched_manifest(&mut current, &next, &request, 102).unwrap(); + assert_eq!(current.segments[&0].encryption.as_ref().unwrap().resource_id, old_key); + let new_key = ¤t.segments[&3].encryption.as_ref().unwrap().resource_id; + assert_ne!(new_key, &old_key); + assert!(current.transient.resources.contains_key(&old_key)); + assert!(current.transient.resources.contains_key(new_key)); +} + +#[tokio::test] +async fn refresh_start_rejects_a_changed_binding_even_if_the_work_generation_is_unchanged() { + let session = test_session(); + let mut request = request_with_origin_work(&session); + { + let mut current = session.write().await; + let owner = current.proxy_session_id.0.clone(); + current.replace_origin_account_binding(Some(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("old"), + owner.clone(), + 1, + 100, + ))); + let snapshot = crate::origin_account_runtime::HlsOriginAccountSnapshot::from_session(¤t); + let generation = snapshot.generation; + request.origin_io.as_mut().unwrap().prepared_account = Some(snapshot); + current.replace_origin_account_binding(Some(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("new"), + owner, + 2, + 100, + ))); + assert_eq!(current.activity.origin_work_generation, generation); + } + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut request, 100).await, + HlsOriginRefreshTriggerOutcome::SessionUnavailable + ); + assert!(!session.read().await.origin_refresh.in_flight); +} diff --git a/backend/hls/src/refresh/tests/archive_master.rs b/backend/hls/src/refresh/tests/archive_master.rs index f3bc2f2af..f720deb54 100644 --- a/backend/hls/src/refresh/tests/archive_master.rs +++ b/backend/hls/src/refresh/tests/archive_master.rs @@ -31,6 +31,7 @@ async fn archive_master_resolution_binds_recovery_to_entry_and_scrubs_redirected headers.insert(header::COOKIE, HeaderValue::from_static("sid=entry-secret")); let context = HlsOriginManifestFetchContext { app_config: test_app_config(), + availability_runtime: None, session: Arc::new(RwLock::new(HlsSession::new( HlsSessionKey::new(1, "12345").with_archive_reference(1_784_898_000), b"secret", diff --git a/backend/hls/src/refresh/tests/headers_redirects.rs b/backend/hls/src/refresh/tests/headers_redirects.rs index c1aa7e8c6..04734b975 100644 --- a/backend/hls/src/refresh/tests/headers_redirects.rs +++ b/backend/hls/src/refresh/tests/headers_redirects.rs @@ -76,6 +76,7 @@ async fn refresh_stores_headers_after_hls_origin_policy() { transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, now_ms: 100, @@ -218,6 +219,7 @@ async fn shared_initial_manifest_automatic_cross_origin_redirect_keeps_identity_ let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers, client: reqwest::Client::builder().no_proxy().build().expect("client"), @@ -276,6 +278,7 @@ async fn shared_initial_manifest_budget_covers_status_decoder_redirect_and_succe let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), diff --git a/backend/hls/src/refresh/tests/origin_binding.rs b/backend/hls/src/refresh/tests/origin_binding.rs index 80534fc1b..94e5dae6a 100644 --- a/backend/hls/src/refresh/tests/origin_binding.rs +++ b/backend/hls/src/refresh/tests/origin_binding.rs @@ -1,8 +1,8 @@ use super::{ assert_stale_switch_generation_rejects_commit, bind_refresh_request_to_app_state, classify_manifest_fetch_failure, commit_fetched_manifest, fetch_and_commit_manifest_with_policy, fetched_manifest, host_from_base_url, - install_published_recovery_binding, manifest_body, manifest_fetch_context, mark_origin_refresh_started, - no_delay_policy, path_has_extension, prepare_cross_host_baseline, publish_ready_test_manifest, refresh_and_commit, + install_published_recovery_binding, manifest_fetch_context, mark_origin_refresh_started, no_delay_policy, + path_has_extension, prepare_cross_host_baseline, publish_ready_test_manifest, refresh_and_commit, request_header_value, resolved_hls_manifest_request_url_from_input, retry_test_manifest_recovery_chain, spawn_test_origin, switch_manifest_body, switch_test_request, test_app_config, test_origin_refresh_request, test_segment_repair_manager, test_session, three_segment_manifest_body, trigger_origin_refresh_sync, @@ -102,7 +102,7 @@ async fn accepted_manifest_commit_stores_provider_session_cookie_separately() { #[tokio::test] async fn provider_failover_status_does_not_count_as_hls_retry() { let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; - let second = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let second = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(100)))).await; let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { name: "demo".into(), urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], @@ -143,6 +143,7 @@ async fn provider_failover_status_does_not_count_as_hls_retry() { transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, @@ -183,7 +184,7 @@ async fn provider_failover_status_does_not_count_as_hls_retry() { assert!(!session.key.stable_value().contains("provider://")); assert!(!session.key.stable_value().contains(first.base_url.as_str())); assert!(!session.key.stable_value().contains(second.base_url.as_str())); - assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.origin_seq_highwater, Some(102)); assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); let metrics = segment_worker_pool.metrics().snapshot(); assert_eq!(metrics.refresh_started, 1); @@ -305,11 +306,7 @@ async fn provider_failover_initial_success_commits_without_hls_host_retry_when_u let second = spawn_test_origin(Arc::new(move |_path| { let hit = second_hits_for_handler.fetch_add(1, Ordering::SeqCst); if hit == 0 { - return ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:102\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n102.ts\n".to_string(), - ); + return (200, Vec::new(), three_segment_manifest_body(100)); } ( 200, @@ -359,6 +356,7 @@ async fn provider_failover_initial_success_commits_without_hls_host_retry_when_u transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, diff --git a/backend/hls/src/refresh/tests/reduced_retry.rs b/backend/hls/src/refresh/tests/reduced_retry.rs new file mode 100644 index 000000000..b0c4deb8d --- /dev/null +++ b/backend/hls/src/refresh/tests/reduced_retry.rs @@ -0,0 +1,338 @@ +use super::*; +use crate::{ + manifest_acceptance::{HlsManifestAcceptanceEpisode, HlsManifestAcceptanceEpisodeStatus}, + session_store::HlsSessionHandle, + SegmentCacheStatus, +}; +use shared::model::HlsManifestRecoveryBurstLevel; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; +use tokio::sync::Mutex; + +const BASELINE: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/baseline.m3u8")); +const CONFLICT: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/conflict.m3u8")); +const REPAIRED: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/repaired.m3u8")); + +async fn commit_baseline(request: &OriginRefreshRequest, context: &HlsOriginManifestFetchContext) { + let baseline = + fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(context)).await.unwrap(); + { + let mut s = request.session.write().await; + let now_ms = tuliprox_core::utils::current_time_millis(); + commit_fetched_manifest(&mut s, &baseline, request, now_ms).unwrap(); + s.activity.last_authorized_media_at_ms = Some(now_ms); + for segment in s.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; + } + s.render_and_store_manifest(now_ms).unwrap(); + assert_eq!(s.origin_seq_highwater, Some(1005)); + assert!(s.established_manifest_recovery_binding().is_some()); + } +} + +async fn commit_observed_recovery_candidate( + request: &OriginRefreshRequest, + fetched: FetchedOriginManifest, + mode: HlsManifestCommitAcceptanceMode, + observations: &Mutex>, +) -> Result { + let before = { + let s = request.session.read().await; + let score = score_manifest_recovery_candidate(&s, &fetched, &manifest_fetch_context(request)).unwrap(); + assert_eq!(score.quality.host_relation, crate::manifest_fetch::HlsManifestOriginRelation::SameRedirectHost); + assert_eq!(score.quality.sequence_relation, HlsManifestSequenceRelation::PlausibleForward); + eprintln!( + "selected candidate host={:?} sequence={:?}", + score.quality.host_relation, score.quality.sequence_relation + ); + s.origin_control.acceptance_episode.clone().unwrap() + }; + let outcome = super::super::commit_manifest_recovery_candidate(request, fetched, mode).await; + let label = match &outcome { + Ok(_) => "committed".to_string(), + Err(HlsManifestCommitError::TimelineRejected { reason }) => reason.status_label(), + Err(error) => format!("{error:?}"), + }; + eprintln!( + "real commit callback={} generation={} full_bursts={} timing={:?}", + label, + before.generation.0, + before.full_bursts_completed, + before.timing() + ); + observations.lock().await.push((label, before)); + outcome +} + +async fn conflict_then_reduced_retry(permanent_conflict: bool) { + let recovery = Arc::new(AtomicBool::new(false)); + let requests = Arc::new(AtomicUsize::new(0)); + let (phase, count) = (Arc::clone(&recovery), Arc::clone(&requests)); + let server = spawn_test_origin(Arc::new(move |_| { + let body = if phase.load(Ordering::Acquire) { + let n = count.fetch_add(1, Ordering::AcqRel) + 1; + eprintln!("recovery HTTP request={n} status=200 permanent={permanent_conflict}"); + if n <= 12 || permanent_conflict { + CONFLICT + } else { + REPAIRED + } + } else { + BASELINE + }; + (200, Vec::new(), body.to_string()) + })) + .await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + let url = format!("{}/index.m3u8", server.base_url); + request.origin_entry = LiveHlsOriginEntry::parse(&url).unwrap(); + request.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast; + let context = manifest_fetch_context(&request); + commit_baseline(&request, &context).await; + recovery.store(true, Ordering::Release); + let callbacks = Arc::new(Mutex::new(Vec::new())); + let observations = Arc::clone(&callbacks); + let request_ref = &request; + let result = tokio::time::timeout( + std::time::Duration::from_secs(15), + retry_hls_origin_manifest_recovery_chain( + &context, + test_manifest_origin_binding(url::Url::parse(&url).unwrap()), + None, + None, + HlsManifestAcceptanceTrigger::RecoveryRequired, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + move |fetched, mode| { + let observations = Arc::clone(&observations); + async move { commit_observed_recovery_candidate(request_ref, fetched, mode, &observations).await } + }, + ), + ) + .await + .unwrap(); + let records = callbacks.lock().await; + let s = session.read().await; + let episode = s.origin_control.acceptance_episode.as_ref().unwrap(); + eprintln!( + "recovery result={:?} requests={} callbacks={} episode={episode:?}", + result.as_ref().map(|_| ()), + requests.load(Ordering::Acquire), + records.len() + ); + assert!( + records[0].0.contains("origin-sequence-resource-conflict"), + "first commit must encounter the real conflict: {}", + records[0].0 + ); + if permanent_conflict { + assert!(result.is_err()); + assert_eq!(requests.load(Ordering::Acquire), 16); + assert_eq!(records.len(), 5, "all reduced candidates must reach the real commit guard"); + assert!(records.iter().all(|(label, _)| label.contains("origin-sequence-resource-conflict"))); + assert!(matches!( + crate::manifest_acceptance::manifest_acceptance_episode_status( + Some(episode), + episode.generation, + request.now_ms + ), + HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { .. } + )); + assert_eq!(s.origin_seq_highwater, Some(1005)); + } else { + assert!( + result.is_ok(), + "repaired single-candidate retry must commit; requests={} callbacks={}", + requests.load(Ordering::Acquire), + records.len() + ); + assert_eq!(requests.load(Ordering::Acquire), 13); + assert_eq!(records.len(), 2); + assert_eq!(records[1].0, "committed"); + assert_eq!(s.origin_seq_highwater, Some(1008)); + } + assert_eq!(episode.full_bursts_completed, 1); + for (_, before) in records.iter() { + assert_eq!(before.generation, episode.generation); + assert_eq!(before.timing(), episode.timing()); + assert_eq!(before.full_bursts_completed, 1); + } +} + +#[tokio::test] +async fn reduced_retry_commits_repaired_same_host_manifest_after_real_resource_conflict() { + conflict_then_reduced_retry(false).await; +} + +#[tokio::test] +async fn reduced_retry_permanent_resource_conflict_exhausts_existing_budget() { + conflict_then_reduced_retry(true).await; +} + +async fn replace_acceptance_episode_with_successor( + session: &HlsSessionHandle, + retry_binding: HlsManifestOriginBinding, + account_handoff: bool, + after_selection: bool, +) -> HlsManifestAcceptanceEpisode { + use crate::short_manifest::{HlsOriginQualification, HlsOriginQualificationBasis}; + + let mut s = session.write().await; + let old = s.origin_control.acceptance_episode.as_ref().unwrap().clone(); + if after_selection { + assert!(old.exhaustion_reason().is_none()); + assert_eq!(old.state, crate::manifest_acceptance::HlsManifestAcceptanceState::Committing); + } else { + assert!(old.exhaustion_reason().is_some()); + } + if account_handoff { + let owner = s.proxy_session_id.0.clone(); + s.handoff_origin_account(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("account-b"), + owner, + 1, + 200, + )); + } else { + s.invalidate_queued_origin_work(); + } + let now = tuliprox_core::utils::current_time_millis(); + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let generation = s.origin_control.begin_acceptance_episode( + now, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &test_acceptance_episode_timing(now, plan), + ); + assert_ne!(generation, old.generation); + s.origin_control.qualification = HlsOriginQualification::Checking { + basis: HlsOriginQualificationBasis::capture(&s), + generation, + target: retry_binding, + }; + s.origin_control.acceptance_episode.clone().unwrap() +} + +async fn superseded_reduced_response_preserves_successor(account_handoff: bool, after_selection: bool) { + use super::support::{spawn_test_origin_with_response_gate, TestOriginResponseGate}; + use tokio::sync::Notify; + + let requests = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&requests); + let gate = Arc::new(TestOriginResponseGate { + request_number: 14, // Baseline plus twelve burst samples, then the reduced retry. + entered: Notify::new(), + release: Notify::new(), + }); + let server = spawn_test_origin_with_response_gate( + Arc::new(move |_| { + let n = count.fetch_add(1, Ordering::AcqRel); + let body = if n == 0 { + BASELINE + } else if n <= 12 { + CONFLICT + } else { + REPAIRED + }; + (200, Vec::new(), body.to_string()) + }), + if after_selection { None } else { Some(Arc::clone(&gate)) }, + ) + .await; + let session = test_session(); + let mut request = super::account_handoff::request_with_origin_work(&session); + request.origin_io.as_mut().unwrap().started_generation = Some(session.read().await.activity.origin_work_generation); + let url = format!("{}/index.m3u8", server.base_url); + request.origin_entry = LiveHlsOriginEntry::parse(&url).unwrap(); + request.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast; + let context = manifest_fetch_context(&request); + commit_baseline(&request, &context).await; + let baseline = session.read().await.origin_to_proxy.clone(); + let callbacks = Arc::new(AtomicUsize::new(0)); + let observed_callbacks = Arc::clone(&callbacks); + let binding = test_manifest_origin_binding(url::Url::parse(&url).unwrap()); + let retry_binding = binding.clone(); + let commit_gate = Arc::clone(&gate); + let recovery = tokio::spawn(async move { + retry_hls_origin_manifest_recovery_chain( + &context, + binding, + None, + None, + HlsManifestAcceptanceTrigger::RecoveryRequired, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, mode| { + let number = observed_callbacks.fetch_add(1, Ordering::AcqRel) + 1; + let gate = Arc::clone(&commit_gate); + let request_ref = &request; + async move { + if after_selection && number == 2 { + gate.entered.notify_one(); + gate.release.notified().await; + } + super::super::commit_manifest_recovery_candidate(request_ref, fetched, mode).await + } + }, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), gate.entered.notified()).await.unwrap(); + let successor = + replace_acceptance_episode_with_successor(&session, retry_binding, account_handoff, after_selection).await; + gate.release.notify_one(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), recovery).await.unwrap().unwrap(); + { + let s = session.read().await; + let actual = s.origin_control.acceptance_episode.as_ref().unwrap(); + eprintln!("REVIEW: after_selection={after_selection} handoff={account_handoff} result={:?} callbacks={} requests={} expected_generation={} actual_generation={} expected_state={:?} actual_state={:?} successor_unchanged={}", + result.as_ref().map(|_| ()), callbacks.load(Ordering::Acquire), requests.load(Ordering::Acquire), + successor.generation.0, actual.generation.0, successor.state, actual.state, actual == &successor); + assert!(actual == &successor, "old commit must leave the successor episode unchanged"); + } + assert!(matches!( + result, + Err(OriginManifestFetchError::RecoveryUnavailable { + reason: HlsManifestRecoveryUnavailableReason::BindingSuperseded + }) + )); + assert_eq!( + callbacks.load(Ordering::Acquire), + if after_selection { 2 } else { 1 }, + "old response must not reach another commit" + ); + assert_eq!(requests.load(Ordering::Acquire), 14, "old chain stops before another attempt"); + let s = session.read().await; + assert_eq!(s.origin_to_proxy, baseline); + assert_eq!( + s.origin_control.acceptance_episode.as_ref(), + Some(&successor), + "old work must leave the pending successor unchanged" + ); + assert_eq!(s.origin_control.qualification.generation(), Some(successor.generation)); +} + +#[tokio::test] +async fn reduced_retry_held_response_cannot_mutate_successor_after_account_handoff() { + superseded_reduced_response_preserves_successor(true, false).await; +} + +#[tokio::test] +async fn reduced_retry_held_response_cannot_mutate_successor_after_work_generation_change() { + superseded_reduced_response_preserves_successor(false, false).await; +} + +#[tokio::test] +async fn review_selected_commit_cannot_mutate_successor_after_account_handoff() { + superseded_reduced_response_preserves_successor(true, true).await; +} + +#[tokio::test] +async fn review_selected_commit_cannot_mutate_successor_after_work_generation_change() { + superseded_reduced_response_preserves_successor(false, true).await; +} diff --git a/backend/hls/src/refresh/tests/retry_policy.rs b/backend/hls/src/refresh/tests/retry_policy.rs index 24e7ce718..5319497c2 100644 --- a/backend/hls/src/refresh/tests/retry_policy.rs +++ b/backend/hls/src/refresh/tests/retry_policy.rs @@ -761,6 +761,7 @@ async fn concurrent_maybe_trigger_origin_refresh_starts_singleflight_once() { transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, @@ -1081,7 +1082,7 @@ async fn compatible_aes_128_manifest_commits_normal_timeline_with_opaque_key_res #[tokio::test] async fn cold_recovery_directive_is_suppressed_until_baseline() { - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(0)))).await; let session = test_session(); let mut request = test_origin_refresh_request(Arc::clone(&session)); request.origin_entry = @@ -1097,7 +1098,7 @@ async fn cold_recovery_directive_is_suppressed_until_baseline() { assert_eq!(manifest_requests, 1); assert_ne!(manifest_requests, plan.total_candidates()); let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.origin_seq_highwater, Some(2)); assert!(session.origin_control.manifest_origin_binding.is_some()); assert!(session.origin_control.acceptance_episode.is_none()); } @@ -1118,6 +1119,7 @@ async fn shared_initial_manifest_decoder_failure_retries_until_success() { let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), @@ -1164,6 +1166,7 @@ async fn shared_initial_manifest_waits_for_next_attempt_base_delay() { let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), @@ -1202,6 +1205,7 @@ async fn shared_initial_manifest_decoder_failures_stop_at_attempt_budget() { let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), @@ -1266,7 +1270,7 @@ async fn shared_refresh_metrics_use_successful_manifest_attempt_count() { .filter(|request| request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 "))) .cloned() .collect::>(); - assert_eq!(manifest_requests.len(), 3); + assert_eq!(manifest_requests.len(), 4, "initial retries plus the configured single-candidate qualification"); assert!(manifest_requests .iter() .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); @@ -1400,7 +1404,7 @@ async fn recovery_unavailable_after_valid_response_preserves_response_evidence() #[tokio::test] async fn cold_start_uses_one_initial_fetch_and_commits_a_fresh_baseline_before_recovery() { let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(0)))).await; let session = test_session(); { let mut session = session.write().await; @@ -1421,7 +1425,7 @@ async fn cold_start_uses_one_initial_fetch_and_commits_a_fresh_baseline_before_r assert_eq!(manifest_requests, 1); assert_ne!(manifest_requests, plan.total_candidates()); let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.origin_seq_highwater, Some(2)); assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); assert!(session.origin_control.acceptance_episode.is_none()); } @@ -1449,3 +1453,38 @@ async fn cold_start_hard_error_is_preserved_by_fetch_policy() { super::super::super::origin_progress::HlsOriginProgressPhase::Cold ); } + +#[tokio::test] +async fn shared_initial_manifest_caps_retry_after_at_half_target_duration() { + for use_manual_redirects in [false, true] { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + if hits_for_handler.fetch_add(1, Ordering::SeqCst) == 0 { + return (502, vec![("Retry-After", "60".to_string())], "retry".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let session = test_session(); + session.write().await.origin_control.record_media_progress(50, 1_000); + let mut request = test_origin_refresh_request(session); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + request.use_manual_redirects = use_manual_redirects; + let context = super::manifest_fetch_context(&request); + let started_at = std::time::Instant::now(); + + let fetched = tokio::time::timeout( + std::time::Duration::from_secs(5), + fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)), + ) + .await + .expect("Retry-After must not stall the manifest fetch for 60 seconds") + .expect("second logical attempt should succeed"); + + assert_eq!(fetched.attempts, 2); + assert_eq!(server.requests.lock().await.len(), 2); + assert!(started_at.elapsed() >= std::time::Duration::from_millis(500)); + } +} diff --git a/backend/hls/src/refresh/tests/short_manifest.rs b/backend/hls/src/refresh/tests/short_manifest.rs new file mode 100644 index 000000000..7353bc17f --- /dev/null +++ b/backend/hls/src/refresh/tests/short_manifest.rs @@ -0,0 +1,669 @@ +use super::*; +use crate::{ + short_manifest::HlsOriginQualification, HlsAccessLeaseId, HlsFreshManifestRequiredReason, HlsManifestRenderer, + SegmentCacheStatus, +}; +use shared::model::HlsManifestRecoveryBurstLevel; +use std::{ + fmt::Write as _, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; +use tuliprox_core::utils::current_time_millis; + +fn playlist(head: u64, count: usize) -> String { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + for offset in 0..count { + writeln!(body, "#EXTINF:2.5,\nsegment-{}.ts", head + u64::try_from(offset).unwrap()).unwrap(); + } + body +} + +fn request_for_origin(server: &TestOriginServer) -> OriginRefreshRequest { + let mut request = test_origin_refresh_request(test_session()); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).unwrap(); + request.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Friendly; + request.now_ms = current_time_millis(); + request +} + +async fn configure_first_ready(request: &OriginRefreshRequest) { + let config = tuliprox_core::model::HlsStartupConfig { + mode: shared::model::HlsStartupMode::FirstReady, + ..Default::default() + }; + request.session.write().await.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 10_000, + worker: Arc::downgrade(&request.segment_worker_pool), + access_leases: Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default())), + revisions: std::collections::BTreeMap::new(), + policy_fixed: true, + store: Arc::new(crate::SegmentRevisionStore::default()), + budget: crate::ProgressiveBudgetManager::new(config), + }); +} + +fn publish_first_ready_head(session: &mut crate::HlsSession, now_ms: u64) { + let head = session.publishable_origin_head_proxy_seq.unwrap(); + let startup = session.startup.as_mut().unwrap(); + let revision = + startup.store.create(session.proxy_session_id.clone(), head, crate::SegmentRevisionKind::Processed).unwrap(); + assert!(revision + .revision() + .complete(crate::CachedSegmentMetadata { path: "first-ready-test.ts".into(), size: 128 })); + startup.revisions.insert(head, revision); + session.segments.get_mut(&head).unwrap().status = + SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: now_ms }; + session.render_and_store_manifest(now_ms).unwrap(); +} + +#[tokio::test] +async fn short_startup_one_segment_waits_for_complete_burst() { startup_with_short_window(1).await; } + +#[tokio::test] +async fn short_startup_two_segments_waits_for_complete_burst() { startup_with_short_window(2).await; } + +async fn startup_with_short_window(count: usize) { + let server = spawn_test_origin(Arc::new(move |_| (200, Vec::new(), playlist(100, count)))).await; + let mut request = request_for_origin(&server); + request.strip.value = 10; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let mut session = request.session.write().await; + assert!(session.allows_short_origin_windows()); + assert_eq!(server.requests.lock().await.len(), 1 + request.manifest_recovery_burst.level.plan().total_candidates()); + assert_eq!(session.origin_control.qualification.generation(), Some(session.origin_control.acceptance_generation)); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + let rendered = session.render_and_store_manifest(request.now_ms).unwrap(); + assert_eq!(rendered.segment_proxy_seqs.len(), count); + assert_eq!(rendered.playlist_duration_ms, u64::try_from(count).unwrap() * 2_500); + assert_eq!(session.initial_prefetch_gap_segments, 0); + assert!(!rendered.body.contains("#EXT-X-ENDLIST")); +} + +#[tokio::test] +async fn short_qualification_survives_ten_refreshes_and_larger_intermediate_windows() { + for short_count in [1, 2] { + let head = Arc::new(AtomicUsize::new(100)); + let count = Arc::new(AtomicUsize::new(short_count)); + let (origin_head, origin_count) = (Arc::clone(&head), Arc::clone(&count)); + let server = spawn_test_origin(Arc::new(move |_| { + ( + 200, + Vec::new(), + playlist( + u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), + origin_count.load(Ordering::SeqCst), + ), + ) + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + let startup_requests = server.requests.lock().await.len(); + for cycle in 1..=10 { + head.store(100 + cycle * 4, Ordering::SeqCst); + count.store(if cycle % 2 == 1 { short_count + 2 } else { short_count }, Ordering::SeqCst); + { + let mut session = request.session.write().await; + session.origin_refresh.next_fetch_allowed_at_ms = 0; + } + request.now_ms = current_time_millis(); + request.access_lease_id = Some(HlsAccessLeaseId(format!("new-lease-{cycle}"))); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::StaleEntryWindow, + }; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let mut session = request.session.write().await; + assert_eq!(session.origin_control.acceptance_generation, generation); + assert!(session.allows_short_origin_windows()); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + let rendered = session.render_and_store_manifest(request.now_ms).unwrap(); + assert!(rendered.segment_proxy_seqs.len() >= count.load(Ordering::SeqCst)); + assert_eq!(server.requests.lock().await.len(), startup_requests + cycle); + } + } +} + +async fn concurrent_refreshes(request: &OriginRefreshRequest, leases: usize) -> usize { + let mut tasks = tokio::task::JoinSet::new(); + for lease in 0..leases { + let mut request = request.clone(); + request.access_lease_id = Some(HlsAccessLeaseId(format!("lease-{lease}"))); + tasks.spawn(trigger_origin_refresh_sync(request)); + } + let mut started = 0; + while let Some(result) = tasks.join_next().await { + started += usize::from(result.unwrap()); + } + started +} + +#[tokio::test] +async fn twenty_leases_share_startup_revalidation_and_normal_freshness_requests() { + let mut request_counts = Vec::new(); + for leases in [1, 20] { + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + assert_eq!(concurrent_refreshes(&request, leases).await, 1); + let initial_count = server.requests.lock().await.len(); + let first_generation = request.session.read().await.origin_control.acceptance_generation; + { + let mut session = request.session.write().await; + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::ExpiredRevalidation); + } + head.store(101, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, leases).await, 1); + let session = request.session.read().await; + assert!(session.allows_short_origin_windows(), "{:?} {:?}", session.origin_control, session.origin_refresh); + assert!(session.origin_control.acceptance_generation.0 > first_generation.0); + assert!(session.fresh_manifest_commit_required.is_none()); + drop(session); + head.store(102, Ordering::SeqCst); + request.session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::StaleEntryWindow, + }; + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, leases).await, 1); + request_counts.push((initial_count, server.requests.lock().await.len())); + } + assert_eq!(request_counts[0], request_counts[1]); +} + +#[tokio::test] +async fn short_failed_qualification_preserves_full_burst_receipt_and_backoff_for_new_leases() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + if origin_calls.fetch_add(1, Ordering::SeqCst) == 0 { + (200, Vec::new(), playlist(100, 1)) + } else { + (500, Vec::new(), String::new()) + } + })) + .await; + let mut request = request_for_origin(&server); + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + assert!(matches!( + request.session.read().await.origin_control.qualification, + HlsOriginQualification::RetryPending { .. } + )); + // The existing failure schedule allows an immediate first retry, then backs off. + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + let before_retry = calls.load(Ordering::SeqCst); + assert!(trigger_origin_refresh_sync(request.clone()).await); + assert_eq!( + calls.load(Ordering::SeqCst), + before_retry + 1, + "failed qualification retries one sample, never a new full burst" + ); + let failed_state = request.session.read().await.origin_refresh.clone(); + request.now_ms = failed_state.next_fetch_allowed_at_ms - 1; + assert_eq!(concurrent_refreshes(&request, 20).await, 0); + let session = request.session.read().await; + assert_eq!(session.origin_refresh, failed_state); + assert_eq!(session.origin_control.acceptance_generation, generation); + assert!(!session.allows_short_origin_windows()); + assert!(session.segments.is_empty()); +} + +#[tokio::test] +async fn later_regular_candidate_wins_before_short_startup_is_published() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + let count = if origin_calls.fetch_add(1, Ordering::SeqCst) < 2 { 1 } else { 3 }; + (200, Vec::new(), playlist(100, count)) + })) + .await; + let request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert_eq!(session.segments.len(), 3); + assert!(!session.allows_short_origin_windows()); + assert_eq!(calls.load(Ordering::SeqCst), 3); +} + +#[tokio::test] +async fn unqualified_short_window_stays_unrenderable_and_unchanged_refresh_does_not_advance_media() { + let session = test_session(); + let request = test_origin_refresh_request(Arc::clone(&session)); + let short = fetched_manifest(&playlist(100, 1)); + let mut session = session.write().await; + commit_fetched_manifest(&mut session, &short, &request, 100).unwrap(); + assert!(!session.allows_short_origin_windows()); + assert!(HlsManifestRenderer::render(&session, 100).is_err()); + let progress = session.origin_control.progress_generation; + commit_fetched_manifest(&mut session, &short, &request, 200).unwrap(); + assert_eq!(session.origin_control.progress_generation, progress); + assert_eq!(session.origin_control.last_media_progress_at_ms, Some(100)); +} + +#[tokio::test] +async fn short_qualification_rejects_stale_epoch_binding_and_acceptance_generation_before_commit() { + use crate::short_manifest::{HlsOriginQualification, HlsOriginQualificationBasis}; + for change in 0..3 { + let handle = test_session(); + let request = test_origin_refresh_request(Arc::clone(&handle)); + let plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + let mut session = handle.write().await; + let generation = session.origin_control.begin_acceptance_episode( + 100, + plan, + HlsManifestAcceptanceTrigger::Observe, + &test_acceptance_episode_timing(100, plan), + ); + let target = test_manifest_origin_binding(url::Url::parse("https://origin.example/live.m3u8").unwrap()); + session.origin_control.qualification = HlsOriginQualification::Checking { + basis: HlsOriginQualificationBasis::capture(&session), + generation, + target: target.clone(), + }; + session.origin_control.acceptance_episode.as_mut().unwrap().record_full_burst(); + match change { + 0 => session.origin_epoch += 1, + 1 => session.origin_control.manifest_origin_binding = Some(target), + 2 => session.origin_control.acceptance_generation.0 += 1, + _ => unreachable!(), + } + let result = commit_fetched_manifest(&mut session, &fetched_manifest(&playlist(100, 1)), &request, 200); + assert!(matches!( + result, + Err(HlsManifestCommitError::TimelineRejected { + reason: HlsManifestRejectLogReason::StagedSwitchInvalidated, + }) + )); + assert!(!session.allows_short_origin_windows()); + assert!(session.segments.is_empty()); + assert_eq!(session.origin_manifest_commits(), 0); + } +} + +struct QualificationLogger(std::sync::Mutex>); + +impl log::Log for QualificationLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { metadata.level() == log::Level::Warn } + fn log(&self, record: &log::Record<'_>) { + if self.enabled(record.metadata()) { + let message = record.args().to_string(); + if message.starts_with("HLS short origin window qualified:") { + self.0.lock().unwrap().push(message); + } + } + } + fn flush(&self) {} +} + +#[tokio::test] +async fn short_window_warning_occurs_once_per_qualification_across_new_leases() { + static LOGGER: QualificationLogger = QualificationLogger(std::sync::Mutex::new(Vec::new())); + log::set_logger(&LOGGER).unwrap(); + log::set_max_level(log::LevelFilter::Warn); + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + request.session = Arc::new(tokio::sync::RwLock::new(crate::HlsSession::new( + crate::HlsSessionKey::new(1, "warning-once"), + b"secret", + current_time_millis(), + ))); + assert!(trigger_origin_refresh_sync(request.clone()).await); + for lease in 0..10 { + request.access_lease_id = Some(HlsAccessLeaseId(format!("warning-lease-{lease}"))); + request.now_ms = current_time_millis(); + request.session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + assert!(trigger_origin_refresh_sync(request.clone()).await); + } + let proxy_session = crate::safe_proxy_session_id(&request.session.read().await.proxy_session_id); + let warnings = || { + LOGGER + .0 + .lock() + .unwrap() + .iter() + .filter(|message| message.contains(&format!("proxy_session={proxy_session}"))) + .cloned() + .collect::>() + }; + let first = warnings(); + assert_eq!(first.len(), 1); + assert!(first[0].contains("RFC 8216 section 6.2.2")); + assert!(first[0].contains(&format!("host={}", crate::hls_origin_log_value("127.0.0.1")))); + assert!(first[0].contains("segments=1 qualification=1 burst=2")); + head.store(101, Ordering::SeqCst); + request.session.write().await.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::ExpiredRevalidation); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request).await); + assert_eq!(warnings().len(), 2); +} + +#[tokio::test] +async fn short_permission_does_not_release_empty_replayed_or_failed_media() { + let server = spawn_test_origin(Arc::new(|_| (200, Vec::new(), playlist(100, 1)))).await; + let request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let mut session = request.session.write().await; + assert!(session.allows_short_origin_windows()); + let proxy_seq = session.publishable_origin_head_proxy_seq.unwrap(); + session.segments.get_mut(&proxy_seq).unwrap().status = SegmentCacheStatus::FailedPermanent { + failed_at_ms: request.now_ms, + status: Some(axum::http::StatusCode::NOT_FOUND), + }; + assert!(HlsManifestRenderer::render(&session, request.now_ms).is_err()); + session.segments.get_mut(&proxy_seq).unwrap().status = + SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + session.render_and_store_manifest(request.now_ms).unwrap(); + let commits = session.origin_manifest_commits(); + let mut replay = fetched_manifest(&playlist(101, 1).replace("segment-101.ts", "segment-100.ts")); + replay.final_manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); + replay.resolved_request_url.clone_from(&replay.final_manifest_url); + replay.redirect_host = None; + assert!(matches!( + commit_fetched_manifest(&mut session, &replay, &request, request.now_ms + 1), + Err(HlsManifestCommitError::TimelineRejected { + reason: HlsManifestRejectLogReason::PublishedResourceReplay { .. } + }) + )); + assert!(session.allows_short_origin_windows()); + assert_eq!(session.origin_manifest_commits(), commits); + let mut empty = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:102\n"); + empty.final_manifest_url = replay.final_manifest_url; + empty.resolved_request_url = replay.resolved_request_url; + empty.redirect_host = None; + assert!(commit_fetched_manifest(&mut session, &empty, &request, request.now_ms + 2).is_err()); + assert_eq!(session.origin_manifest_commits(), commits); +} + +#[tokio::test] +async fn expired_manifest_requires_new_qualification_without_an_explicit_fresh_requirement_flag() { + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + { + let mut session = request.session.write().await; + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + session.render_and_store_manifest(request.now_ms).unwrap(); + assert!(session.last_normal_manifest_commit_identity().is_some()); + session.last_rendered_manifest = None; + } + assert!(request.session.read().await.fresh_manifest_commit_required.is_none()); + head.store(101, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert!(session.origin_control.acceptance_generation.0 > generation.0); + assert!(session.allows_short_origin_windows()); + assert_eq!(server.requests.lock().await.len(), 6); +} + +#[tokio::test] +async fn recovered_origin_requalifies_after_expired_episode_under_shared_backoff() { + let phase = Arc::new(AtomicUsize::new(0)); + let origin_phase = Arc::clone(&phase); + let server = spawn_test_origin(Arc::new(move |_| match origin_phase.load(Ordering::SeqCst) { + 0 => (200, Vec::new(), playlist(100, 1)), + 1 => (500, Vec::new(), String::new()), + _ => (200, Vec::new(), playlist(101, 1)), + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + { + let mut session = request.session.write().await; + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + session.render_and_store_manifest(request.now_ms).unwrap(); + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::ExpiredRevalidation); + } + phase.store(1, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let failed_generation = { + let mut session = request.session.write().await; + let previous = session.origin_control.acceptance_episode.take().unwrap(); + let mut expired = crate::manifest_acceptance::HlsManifestAcceptanceEpisode::new( + previous.generation, + 1, + previous.burst_plan, + previous.trigger(), + &test_acceptance_episode_timing(1, previous.burst_plan), + ); + expired.record_full_burst(); + expired.state = previous.state; + expired.observed_landscape = previous.observed_landscape; + assert!(expired.timing().acceptance_deadline.as_millis_since_epoch() < current_time_millis()); + session.origin_control.acceptance_episode = Some(expired); + assert_eq!(session.origin_seq_highwater, Some(100)); + session.origin_control.acceptance_generation + }; + let requests_before_retry = server.requests.lock().await.len(); + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + assert_eq!(server.requests.lock().await.len(), requests_before_retry + 1); + assert_eq!(request.session.read().await.origin_control.acceptance_generation, failed_generation); + let retry_at_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + request.now_ms = retry_at_ms - 1; + phase.store(2, Ordering::SeqCst); + assert_eq!(concurrent_refreshes(&request, 20).await, 0); + assert_eq!(server.requests.lock().await.len(), requests_before_retry + 1); + request.now_ms = retry_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + let session = request.session.read().await; + assert_eq!(session.origin_seq_highwater, Some(101)); + assert_eq!(session.origin_control.acceptance_generation.0, failed_generation.0 + 1); + assert!(session.allows_short_origin_windows()); + assert_eq!(session.origin_refresh.consecutive_failures, 0); + assert!(session.fresh_manifest_commit_required.is_none()); + assert_eq!(server.requests.lock().await.len(), requests_before_retry + 4); +} + +#[tokio::test] +async fn new_leases_reuse_origin_qualification_before_first_publication() { + let server = spawn_test_origin(Arc::new(|_| (200, Vec::new(), playlist(100, 1)))).await; + let mut request = request_for_origin(&server); + configure_first_ready(&request).await; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = { + let session = request.session.read().await; + assert!(session.origin_is_qualified()); + assert!(session.last_rendered_manifest.is_none()); + assert!(session.last_normal_manifest_commit_identity().is_none()); + assert!(session.segments.values().all(|segment| !matches!(segment.status, SegmentCacheStatus::Ready { .. }))); + session.origin_control.acceptance_generation + }; + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + assert_eq!(request.session.read().await.origin_control.acceptance_generation, generation); + assert_eq!(server.requests.lock().await.len(), 4); +} + +#[tokio::test] +async fn first_ready_revalidation_is_shared_until_its_publication_then_can_expire_again() { + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + configure_first_ready(&request).await; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let previous_commit = { + let mut session = request.session.write().await; + publish_first_ready_head(&mut session, request.now_ms); + session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0; + assert!(!crate::manifest_commit::has_servable_committed_manifest(&session, request.now_ms)); + session.last_normal_manifest_commit_identity().unwrap() + }; + head.store(101, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + { + let session = request.session.read().await; + assert_eq!(session.startup_mode(), shared::model::HlsStartupMode::FirstReady); + assert_eq!(session.origin_control.acceptance_generation.0, 2); + assert_eq!(session.origin_seq_highwater, Some(101)); + assert!(session.origin_is_qualified()); + assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit)); + assert!(!crate::manifest_commit::has_servable_committed_manifest(&session, request.now_ms)); + } + assert_eq!(server.requests.lock().await.len(), 5); + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + { + let mut session = request.session.write().await; + assert_eq!(session.origin_control.acceptance_generation.0, 2); + assert!(session.origin_is_qualified()); + assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit)); + assert_eq!(server.requests.lock().await.len(), 6); + publish_first_ready_head(&mut session, request.now_ms); + assert!(session.last_normal_manifest_commit_identity().unwrap() > previous_commit); + assert!(crate::manifest_commit::has_servable_committed_manifest(&session, request.now_ms)); + session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0; + } + head.store(102, Ordering::SeqCst); + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert_eq!(session.origin_control.acceptance_generation.0, 3); + assert_eq!(session.origin_seq_highwater, Some(102)); + assert!(session.origin_is_qualified()); + assert_eq!(server.requests.lock().await.len(), 8); +} + +#[tokio::test] +async fn genuine_lifecycle_revalidation_replaces_failed_startup_once_and_shares_its_failure() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + if origin_calls.fetch_add(1, Ordering::SeqCst) == 0 { + (200, Vec::new(), playlist(100, 1)) + } else { + (500, Vec::new(), String::new()) + } + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + request + .session + .write() + .await + .require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + }; + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + let revalidated = request.session.read().await.origin_control.acceptance_generation; + assert!(revalidated.0 > generation.0); + let requests_after_revalidation = calls.load(Ordering::SeqCst); + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + assert_eq!(calls.load(Ordering::SeqCst), requests_after_revalidation + 1); + assert_eq!(request.session.read().await.origin_control.acceptance_generation, revalidated); +} + +#[tokio::test] +async fn segment_count_change_preserves_regular_qualification_without_starting_a_new_burst() { + let count = Arc::new(AtomicUsize::new(3)); + let origin_count = Arc::clone(&count); + let server = spawn_test_origin(Arc::new(move |_| { + let count = origin_count.load(Ordering::SeqCst); + (200, Vec::new(), playlist(if count == 3 { 100 } else { 104 }, count)) + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + assert!(request.session.read().await.origin_is_qualified()); + assert!(!request.session.read().await.allows_short_origin_windows()); + count.store(1, Ordering::SeqCst); + request.session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::StaleEntryWindow }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert_eq!(session.origin_control.acceptance_generation, generation); + assert!(session.origin_is_qualified()); + assert!(!session.allows_short_origin_windows()); + assert_eq!(server.requests.lock().await.len(), 2); +} + +#[tokio::test] +async fn short_startup_preserves_first_response_provider_cookie_for_burst_and_media_fetches() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + let headers = if origin_calls.fetch_add(1, Ordering::SeqCst) == 0 { + vec![("Set-Cookie", "sid=short-origin; Path=/; HttpOnly".to_string())] + } else { + Vec::new() + }; + (200, headers, playlist(100, 1)) + })) + .await; + let request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let raw_requests = server.raw_requests.lock().await; + assert_eq!(raw_requests.len(), 3); + assert!(raw_requests[1..] + .iter() + .all(|request| request_header_value(request, "cookie") == Some("sid=short-origin"))); + let session = request.session.read().await; + assert!(session.allows_short_origin_windows()); + assert_eq!(session.origin_provider_session_headers.get(axum::http::header::COOKIE).unwrap(), "sid=short-origin"); + assert!(!session.origin_request_headers.contains_key(axum::http::header::COOKIE)); +} diff --git a/backend/hls/src/refresh/tests/support.rs b/backend/hls/src/refresh/tests/support.rs index 07323d3cc..9c308c7b1 100644 --- a/backend/hls/src/refresh/tests/support.rs +++ b/backend/hls/src/refresh/tests/support.rs @@ -167,6 +167,7 @@ pub(in crate::refresh::tests) async fn refresh_session_with_origin_body(body: &' transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, @@ -204,6 +205,19 @@ impl Drop for TestOriginServer { } pub(in crate::refresh::tests) async fn spawn_test_origin(handler: TestOriginHandler) -> TestOriginServer { + spawn_test_origin_with_response_gate(handler, None).await +} + +pub(in crate::refresh::tests) struct TestOriginResponseGate { + pub request_number: usize, + pub entered: Notify, + pub release: Notify, +} + +pub(in crate::refresh::tests) async fn spawn_test_origin_with_response_gate( + handler: TestOriginHandler, + gate: Option>, +) -> TestOriginServer { let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); let addr = listener.local_addr().expect("local addr"); let requests = Arc::new(Mutex::new(Vec::new())); @@ -218,6 +232,7 @@ pub(in crate::refresh::tests) async fn spawn_test_origin(handler: TestOriginHand let requests = Arc::clone(&requests_for_task); let raw_requests = Arc::clone(&raw_requests_for_task); let handler = Arc::clone(&handler); + let gate = gate.clone(); tokio::spawn(async move { let mut buf = vec![0_u8; 4096]; let mut used = 0_usize; @@ -239,9 +254,17 @@ pub(in crate::refresh::tests) async fn spawn_test_origin(handler: TestOriginHand let request = String::from_utf8_lossy(&buf[..used]).into_owned(); let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/").to_string(); - requests.lock().await.push(path.clone()); + let request_number = { + let mut requests = requests.lock().await; + requests.push(path.clone()); + requests.len() + }; raw_requests.lock().await.push(request); let (status, headers, body) = handler(path); + if let Some(gate) = gate.filter(|gate| gate.request_number == request_number) { + gate.entered.notify_one(); + gate.release.notified().await; + } let reason = match status { 200 => "OK", 302 => "Found", @@ -349,6 +372,7 @@ pub(in crate::refresh::tests) fn test_origin_refresh_request(session: Arc bool { fn select_window(session: &HlsSession, render_gap_segments: usize, tail_policy: RenderTailPolicy) -> Option> { let head_seq = session.publishable_origin_head_proxy_seq?; + let minimum_visible_segments = if session.allows_short_origin_windows() { 1 } else { MIN_VISIBLE_SEGMENTS }; if let Some(startup) = &session.startup { // The head always needs body bytes; only optimistic successors may still be waiting for theirs. if !startup.revisions.get(&head_seq).is_some_and(|revision| revision.revision().is_publishable()) { @@ -272,7 +273,7 @@ fn select_window(session: &HlsSession, render_gap_segments: usize, tail_policy: } window.push(seq); } - if window.len() < MIN_VISIBLE_SEGMENTS { + if window.len() < minimum_visible_segments { return None; } // Fast-start state lives as long as the session, so live slides render here too. @@ -332,7 +333,7 @@ fn select_window(session: &HlsSession, render_gap_segments: usize, tail_policy: } retain_published_predecessors(session, &mut window); - if window.len() < MIN_VISIBLE_SEGMENTS { + if window.len() < minimum_visible_segments { return None; } // Lease admission enforces the READY-duration reserve. The renderer only diff --git a/backend/hls/src/response/tests.rs b/backend/hls/src/response/tests.rs index 12852e7d0..38a23ec1a 100644 --- a/backend/hls/src/response/tests.rs +++ b/backend/hls/src/response/tests.rs @@ -100,6 +100,7 @@ impl CachedSegmentFixture { host_local_index: u32::try_from(proxy_seq).expect("test proxy sequence"), }, proxy_seq, + media_start_ms: proxy_seq * 4_000, duration_ms: 4_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/hls/src/runtime_custom_tail.rs b/backend/hls/src/runtime_custom_tail.rs index 77e9120e6..d04509367 100644 --- a/backend/hls/src/runtime_custom_tail.rs +++ b/backend/hls/src/runtime_custom_tail.rs @@ -1,7 +1,7 @@ use super::{ availability::{ commit_prepared_runtime_custom_tail, commit_terminal_tail_if_lease_reserve_requires_cutover, - HlsTerminalResolution, + terminal_bundle_failure_compatibility, terminal_bundle_incompatibility, HlsTerminalResolution, }, hls_ctx::HlsCtx, lease::{new_hls_access_lease_id, HlsRuntimePolicyRevocation, HlsRuntimePolicyRevocationOutcome}, @@ -12,7 +12,8 @@ use super::{ session_store::HlsSessionHandle, terminal_tail::{ snapshot_terminal_media_asset, terminal_media_asset_identity, HlsLeasePlaybackMode, HlsTerminalAssetIdentity, - HlsTerminalMediaAsset, HlsTerminalTailCompatibility, HLS_TERMINAL_TAIL_SEGMENT_COUNT, + HlsTerminalMediaAsset, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, }, HlsAccessLeaseId, ProxySessionId, }; @@ -359,7 +360,9 @@ async fn resolve_hls_runtime_custom_tail_after_revocation( return HlsRuntimeCustomTailOutcome::NoLiveLeaseContext; }; match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) => return HlsRuntimeCustomTailOutcome::AlreadyCommitted, + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalCustomResponse(_) => { + return HlsRuntimeCustomTailOutcome::AlreadyCommitted + } HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { return HlsRuntimeCustomTailOutcome::Superseded; } @@ -580,6 +583,7 @@ impl HlsStandaloneCustomAccessStore { username: Arc::clone(&entry.access.username), shared_lease: entry.access.shared_lease.clone(), bytes, + terminal_generation: None, }) } @@ -605,6 +609,7 @@ pub struct HlsStandaloneCustomSegmentAccess { pub username: Arc, pub shared_lease: Option, pub bytes: bytes::Bytes, + pub(crate) terminal_generation: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -620,52 +625,92 @@ pub fn standalone_custom_asset_fingerprint(identity: HlsTerminalAssetIdentity) - /// Resolves one immutable prepared standalone segment without scanning or /// rewriting MPEG-TS at the HTTP request boundary. -pub fn resolve_hls_standalone_custom_segment( +pub async fn resolve_hls_standalone_custom_segment( ctx: &HlsCtx, lease_id: &HlsAccessLeaseId, asset_fingerprint: &str, index: u16, now_ms: u64, ) -> Result { - ctx.hls_proxy.resolve_standalone_custom_segment(lease_id, asset_fingerprint, index, now_ms) + ctx.hls_proxy.resolve_standalone_custom_segment(lease_id, asset_fingerprint, index, now_ms).await } -async fn await_prepared_standalone_bundle( +pub(crate) async fn await_prepared_standalone_bundle( ctx: &HlsCtx, asset: Arc, key: HlsPreparedTerminalBundleKey, -) -> Option> { +) -> Result, HlsTerminalTailCompatibility> { let state = ctx.hls_proxy.prepared_terminal_bundle_state(key).unwrap_or_else(|| { ctx.hls_proxy.start_prepared_terminal_bundle(asset, key.target_duration_ms, key.segment_count) }); - match state { - HlsPreparedTerminalBundleState::Ready { bundle } if bundle.matches_key_and_shape(key) => Some(bundle), - HlsPreparedTerminalBundleState::Preparing { .. } => { - let HlsPreparedTerminalBundleObservation::Flight(ticket) = - ctx.hls_proxy.observe_prepared_terminal_bundle(key) - else { - return None; - }; - let timeout_ms = ctx.hls_proxy.origin_manifest_timeout_ms().max(1); - match tokio::time::timeout(Duration::from_millis(timeout_ms), ticket.wait()).await { - Ok(HlsPreparedTerminalBundleCompletion::Ready { bundle }) if bundle.matches_key_and_shape(key) => { - Some(bundle) + let state = match state { + HlsPreparedTerminalBundleState::Preparing { key: preparing_key } if preparing_key == key => { + match ctx.hls_proxy.observe_prepared_terminal_bundle(key) { + HlsPreparedTerminalBundleObservation::Flight(ticket) => { + let timeout_ms = ctx.hls_proxy.origin_manifest_timeout_ms().max(1); + let completion = tokio::time::timeout(Duration::from_millis(timeout_ms), ticket.wait()) + .await + .map_err(|_| HlsTerminalTailCompatibility::TerminalMediaNotReady)?; + return prepared_standalone_completion_result(completion, key); + } + HlsPreparedTerminalBundleObservation::Settled(state) => state, + HlsPreparedTerminalBundleObservation::Missing => { + return Err(HlsTerminalTailCompatibility::TerminalMediaNotReady) } - Ok( - HlsPreparedTerminalBundleCompletion::Ready { .. } - | HlsPreparedTerminalBundleCompletion::Failed { .. } - | HlsPreparedTerminalBundleCompletion::Incompatible { .. } - | HlsPreparedTerminalBundleCompletion::FlightReplaced { .. }, - ) - | Err(_) => None, } } - HlsPreparedTerminalBundleState::Ready { .. } - | HlsPreparedTerminalBundleState::Failed { .. } - | HlsPreparedTerminalBundleState::Incompatible { .. } => None, + state => state, + }; + prepared_standalone_bundle_result(state, key) +} + +// Both cached states and completed flights retain the same key-bound compatibility diagnosis. +fn prepared_standalone_bundle_result( + state: HlsPreparedTerminalBundleState, + key: HlsPreparedTerminalBundleKey, +) -> Result, HlsTerminalTailCompatibility> { + match state { + HlsPreparedTerminalBundleState::Ready { bundle } if bundle.matches_key_and_shape(key) => Ok(bundle), + HlsPreparedTerminalBundleState::Ready { .. } => Err(HlsTerminalTailCompatibility::AssetRevisionMismatch), + HlsPreparedTerminalBundleState::Failed { key: actual_key, reason } => { + Err(terminal_bundle_failure_compatibility(actual_key, key, reason)) + } + HlsPreparedTerminalBundleState::Incompatible { key: actual_key, reason } => Err(if actual_key == key { + terminal_bundle_incompatibility(reason) + } else { + HlsTerminalTailCompatibility::AssetRevisionMismatch + }), + HlsPreparedTerminalBundleState::Preparing { key: actual_key } => Err(if actual_key == key { + HlsTerminalTailCompatibility::TerminalMediaNotReady + } else { + HlsTerminalTailCompatibility::AssetRevisionMismatch + }), } } +fn prepared_standalone_completion_result( + completion: HlsPreparedTerminalBundleCompletion, + key: HlsPreparedTerminalBundleKey, +) -> Result, HlsTerminalTailCompatibility> { + let state = match completion { + HlsPreparedTerminalBundleCompletion::Ready { bundle } => HlsPreparedTerminalBundleState::Ready { bundle }, + HlsPreparedTerminalBundleCompletion::Failed { key, reason } => { + HlsPreparedTerminalBundleState::Failed { key, reason } + } + HlsPreparedTerminalBundleCompletion::Incompatible { key, reason } => { + HlsPreparedTerminalBundleState::Incompatible { key, reason } + } + HlsPreparedTerminalBundleCompletion::FlightReplaced { key: actual_key, generation } => { + return Err(if actual_key == key && generation > 0 { + HlsTerminalTailCompatibility::TerminalMediaNotReady + } else { + HlsTerminalTailCompatibility::AssetRevisionMismatch + }) + } + }; + prepared_standalone_bundle_result(state, key) +} + pub async fn build_hls_standalone_custom_plan( ctx: &HlsCtx, base_url: &str, @@ -681,12 +726,9 @@ pub async fn build_hls_standalone_custom_plan( target_duration_ms, segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT, }; - let Some(prepared_bundle) = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), prepared_key).await - else { - return Err(HlsRuntimeCustomTailCompatibility::Incompatible( - HlsTerminalTailCompatibility::TerminalMediaNotReady, - )); - }; + let prepared_bundle = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), prepared_key) + .await + .map_err(HlsRuntimeCustomTailCompatibility::Incompatible)?; let key = HlsStandaloneCustomPlanKey { reason, asset: media, segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT }; let asset_fingerprint = standalone_custom_asset_fingerprint(media); let extinf = format_hls_duration_ms(snapshot.asset.duration_ms()); @@ -871,4 +913,78 @@ mod tests { ); assert!(store.resolve(&HlsAccessLeaseId("lease-1".to_string()), "0000000000000001", 0, 1,).is_ok()); } + #[test] + fn standalone_preparation_preserves_cached_and_completed_incompatibility() { + use crate::prepared_terminal_bundle::{ + HlsPreparedTerminalBundleBuildError, HlsPreparedTerminalBundleFailure, + HlsPreparedTerminalBundleIncompatibility, + }; + let key = standalone_access_entry("preparation-error", 7, 100).plan.prepared_bundle.key; + let reason = + HlsPreparedTerminalBundleIncompatibility::TargetDurationExceeded { asset_ms: 12_000, target_ms: 1_000 }; + let expected = HlsTerminalTailCompatibility::TargetDurationExceeded { asset_ms: 12_000, target_ms: 1_000 }; + for state in [ + HlsPreparedTerminalBundleState::Incompatible { key, reason }, + HlsPreparedTerminalBundleState::Failed { + key, + reason: HlsPreparedTerminalBundleFailure::Build(HlsPreparedTerminalBundleBuildError::Incompatible( + reason, + )), + }, + ] { + assert_eq!(prepared_standalone_bundle_result(state, key).unwrap_err(), expected); + } + for completion in [ + HlsPreparedTerminalBundleCompletion::Incompatible { key, reason }, + HlsPreparedTerminalBundleCompletion::Failed { + key, + reason: HlsPreparedTerminalBundleFailure::Build(HlsPreparedTerminalBundleBuildError::Incompatible( + reason, + )), + }, + ] { + assert_eq!(prepared_standalone_completion_result(completion, key).unwrap_err(), expected); + } + } + + #[test] + fn standalone_preparation_preserves_invalid_asset_and_rejects_stale_keys() { + use crate::prepared_terminal_bundle::{ + HlsPreparedTerminalBundleBuildError, HlsPreparedTerminalBundleFailure, + HlsPreparedTerminalBundleIncompatibility, + }; + let key = standalone_access_entry("invalid-preparation", 7, 100).plan.prepared_bundle.key; + let invalid = + HlsPreparedTerminalBundleFailure::Build(HlsPreparedTerminalBundleBuildError::PublishedBundleShapeMismatch); + assert_eq!( + prepared_standalone_completion_result( + HlsPreparedTerminalBundleCompletion::Failed { key, reason: invalid }, + key + ) + .unwrap_err(), + HlsTerminalTailCompatibility::InvalidAsset + ); + let stale_key = HlsPreparedTerminalBundleKey { target_duration_ms: 2_000, ..key }; + for completion in [ + HlsPreparedTerminalBundleCompletion::Failed { key: stale_key, reason: invalid }, + HlsPreparedTerminalBundleCompletion::Incompatible { + key: stale_key, + reason: HlsPreparedTerminalBundleIncompatibility::ZeroTargetDuration, + }, + HlsPreparedTerminalBundleCompletion::FlightReplaced { key: stale_key, generation: 1 }, + ] { + assert_eq!( + prepared_standalone_completion_result(completion, key).unwrap_err(), + HlsTerminalTailCompatibility::AssetRevisionMismatch + ); + } + assert_eq!( + prepared_standalone_completion_result( + HlsPreparedTerminalBundleCompletion::FlightReplaced { key, generation: 1 }, + key + ) + .unwrap_err(), + HlsTerminalTailCompatibility::TerminalMediaNotReady + ); + } } diff --git a/backend/hls/src/segment_fetcher.rs b/backend/hls/src/segment_fetcher.rs index c4c12a54f..26df1a71f 100644 --- a/backend/hls/src/segment_fetcher.rs +++ b/backend/hls/src/segment_fetcher.rs @@ -1,20 +1,19 @@ #![allow(clippy::large_futures)] use super::{ - begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, classify_hls_backpressure, - fetch_hls_transient_origin_response_with_attempt_prepare, finish_hls_origin_account_io, hls_object_body_deadline, + build_hls_origin_resource_headers, classify_hls_backpressure, + fetch_hls_transient_origin_response_with_attempt_prepare, hls_object_body_deadline, run_hls_origin_resource_retry_loop_with_attempt_prepare, transient::transient_object_expires_at, - CachedSegmentMetadata, HlsAccessLeaseId, HlsAccessLeaseStore, HlsBackpressureState, - HlsBoundAccountAcquireErrorKind, HlsCacheCapacityRevision, HlsCacheMetrics, HlsCacheObjectKey, - HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceBodyDeadline, - HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, - HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSegmentEncryption, HlsSegmentFailureObject, - HlsSegmentFailureTransition, HlsSegmentFile, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, - HlsSegmentRepairSource, HlsSessionHandle, HlsTransientObjectFetchFinalizer, HlsTransientOriginFetchRequest, - OriginSegmentFetchRef, OriginSegmentKey, ProxySessionId, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, - SegmentFetchPriority, StagedCacheObject, TransientObjectFetchDecision, TransientObjectFetchToken, - TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, TransientResourceId, - TransientResourceKind, TransientResourceRef, + CachedSegmentMetadata, HlsAccessLeaseId, HlsAccessLeaseStore, HlsBackpressureState, HlsCacheCapacityRevision, + HlsCacheMetrics, HlsCacheObjectKey, HlsOriginByteRangeExpectation, HlsOriginIoContext, + HlsOriginResourceBodyDeadline, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, + HlsRepairRenderedObjectId, HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSegmentEncryption, + HlsSegmentFailureObject, HlsSegmentFailureTransition, HlsSegmentFile, HlsSegmentRepairManager, + HlsSegmentRepairObjectContext, HlsSegmentRepairSource, HlsSessionHandle, HlsTransientObjectFetchFinalizer, + HlsTransientOriginFetchRequest, OriginSegmentFetchRef, OriginSegmentKey, ProxySessionId, SegmentCacheKey, + SegmentCacheStatus, SegmentEntry, SegmentFetchPriority, StagedCacheObject, TransientObjectFetchDecision, + TransientObjectFetchToken, TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, + TransientResourceId, TransientResourceKind, TransientResourceRef, }; use arc_swap::ArcSwap; use axum::http::HeaderMap; @@ -91,6 +90,18 @@ pub struct HlsSegmentWorkerPool { access_leases: Arc>, metrics: Arc, availability_reevaluations: Option>, + #[cfg(any(test, feature = "test-support"))] + origin_prepare_barrier: std::sync::Mutex>>, +} + +/// One-shot observation points around scheduler account preparation, before any queue is consumed. +#[cfg(any(test, feature = "test-support"))] +#[derive(Default)] +pub struct HlsSegmentPrepareBarrier { + pub entered: tokio::sync::Notify, + pub prepare: tokio::sync::Notify, + pub prepared: tokio::sync::Notify, + pub proceed: tokio::sync::Notify, } #[cfg(test)] @@ -104,6 +115,7 @@ mod origin; mod policy; mod scheduler; mod startup_fill; +mod startup_write; mod switch_staging; #[cfg(test)] diff --git a/backend/hls/src/segment_fetcher/commit.rs b/backend/hls/src/segment_fetcher/commit.rs index e7c076cdb..9564cfe87 100644 --- a/backend/hls/src/segment_fetcher/commit.rs +++ b/backend/hls/src/segment_fetcher/commit.rs @@ -19,11 +19,7 @@ pub(super) struct SegmentFetchCommit { pub(super) generation_valid: bool, } -#[derive(Clone, Copy)] -pub(super) struct SegmentOriginWorkFinish { - pub(super) generation_valid: bool, - pub(super) refresh_reservation: bool, -} +pub(super) use crate::origin_attempt::HlsMediaOriginWorkFinish as SegmentOriginWorkFinish; pub(super) fn segment_fetch_attempt_matches(entry: &SegmentEntry, snapshot: &SegmentFetchSnapshot) -> bool { entry.origin_key == snapshot.origin_key diff --git a/backend/hls/src/segment_fetcher/demand.rs b/backend/hls/src/segment_fetcher/demand.rs index cef185b08..fdc27a052 100644 --- a/backend/hls/src/segment_fetcher/demand.rs +++ b/backend/hls/src/segment_fetcher/demand.rs @@ -125,6 +125,27 @@ impl HlsSegmentWorkerPool { { return None; } + let needs_key_io = { + let session = context.session.read().await; + session.segments.values().filter_map(|entry| entry.encryption.as_ref()).any(|encryption| { + session + .transient + .ready_key_object_valid_until_ms( + &proxy_session_id, + &encryption.resource_id, + &encryption.resource_extension, + now_ms, + ) + .is_none() + }) + }; + if needs_key_io + && crate::origin_account_runtime::prepare_media_origin(context.origin_io.as_ref(), &context.session) + .await + .is_err() + { + return None; + } let mut session = context.session.write().await; if session.is_gc_marked_for_removal() || session.active_segment_fetches >= policy.max_session_segment_fetches { return None; @@ -186,7 +207,9 @@ impl HlsSegmentWorkerPool { } } drop(permit); - if result.is_ok() { + if result.is_ok() + || context.session.read().await.origin_handoff_ready_after(snapshot.binding.origin_work_generation) + { self.schedule_wake(context, current_time_millis()); } } diff --git a/backend/hls/src/segment_fetcher/key_dependency.rs b/backend/hls/src/segment_fetcher/key_dependency.rs index 18fce0043..f84a6a5b2 100644 --- a/backend/hls/src/segment_fetcher/key_dependency.rs +++ b/backend/hls/src/segment_fetcher/key_dependency.rs @@ -80,6 +80,19 @@ pub(super) fn select_key_dependency( { return SegmentKeyDependencySelection::Unavailable; } + if !session.transient.origin_resource_is_fetchable(&resource, now_ms) + && session + .transient + .ready_key_object_valid_until_ms( + proxy_session_id, + &encryption.resource_id, + &encryption.resource_extension, + now_ms, + ) + .is_none() + { + return SegmentKeyDependencySelection::Unavailable; + } let resource_file = TransientResourceFile { resource_id: encryption.resource_id.clone(), extension: encryption.resource_extension.clone(), @@ -190,8 +203,15 @@ pub(super) async fn fetch_segment_key_dependency_into_cache( token.clone(), 1_000, ); - let (staged, origin_work) = - stage_segment_key_dependency(context, policy, &token, &resource, &resource_file).await?; + let (staged, origin_work) = stage_segment_key_dependency( + context, + policy, + &token, + &resource, + &resource_file, + binding.origin_work_generation, + ) + .await?; let generation_valid = origin_work.generation_valid && { let session = context.session.read().await; segment_key_dependency_generation_matches( @@ -249,11 +269,12 @@ async fn stage_segment_key_dependency( token: &TransientObjectFetchToken, resource: &TransientResourceRef, resource_file: &TransientResourceFile, + expected_generation: u64, ) -> Result<(StagedCacheObject, SegmentOriginWorkFinish), SegmentFetchError> { let request = HlsTransientOriginFetchRequest { resolved_origin_uri: resource.resolved_origin_uri.clone(), origin_headers: context.headers.clone(), - origin_provider_session_headers: context.origin_provider_session_headers.clone(), + origin_provider_session_headers: context.session.read().await.origin_provider_session_headers.clone(), range_header: None, resource_file: resource_file.clone(), resource_kind: TransientResourceKind::Key, @@ -268,12 +289,22 @@ async fn stage_segment_key_dependency( super::super::HlsLogIdentity::from_session(&session) }, }; + let resource_for_prepare = resource.clone(); let prepare_context = context.clone(); let prepare_policy = policy.clone(); let response = fetch_hls_transient_origin_response_with_attempt_prepare(request, move |_attempt| { let context = prepare_context.clone(); let policy = prepare_policy.clone(); - async move { prepare_segment_origin_attempt(context, policy).await }.boxed() + let resource = resource_for_prepare.clone(); + async move { + let guard = prepare_segment_origin_attempt(context.clone(), policy, expected_generation).await?; + if !context.session.read().await.transient.origin_resource_is_fetchable(&resource, current_time_millis()) { + finish_segment_origin_attempt(context, guard).await; + return Err(SegmentFetchError::Timeout); + } + Ok(guard) + } + .boxed() }) .await; let response = match response { diff --git a/backend/hls/src/segment_fetcher/origin.rs b/backend/hls/src/segment_fetcher/origin.rs index 0eb8e1754..fa17ab695 100644 --- a/backend/hls/src/segment_fetcher/origin.rs +++ b/backend/hls/src/segment_fetcher/origin.rs @@ -1,16 +1,15 @@ use super::{ - begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, commit_startup_response, - ensure_segment_key_dependency_ready, finish_hls_origin_account_io, hls_object_body_deadline, - reliable_decoded_content_length, run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, - HlsBoundAccountAcquireErrorKind, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, - HlsOriginResourceBodyDeadline, HlsOriginResourceClients, HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, - HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, - SegmentFetchCommit, SegmentFetchContext, SegmentFetchError, SegmentFetchPolicy, SegmentFetchSnapshot, - SegmentOriginWorkFinish, + build_hls_origin_resource_headers, commit_startup_response, ensure_segment_key_dependency_ready, + hls_object_body_deadline, reliable_decoded_content_length, run_hls_origin_resource_retry_loop_with_attempt_prepare, + CachedSegmentMetadata, HlsOriginByteRangeExpectation, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, + HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, HlsResourceFetchKind, HlsResourceFetchSource, + HlsSegmentRepairObjectContext, HlsSegmentRepairSource, SegmentFetchCommit, SegmentFetchContext, SegmentFetchError, + SegmentFetchPolicy, SegmentFetchSnapshot, SegmentOriginWorkFinish, }; +use crate::origin_attempt::{prepare_media_origin_attempt, HlsMediaOriginAttemptGuard as SegmentOriginAttemptGuard}; use axum::http::HeaderMap; use futures::FutureExt; -use tuliprox_core::utils::{content_coding::DecodedHttpResponse, current_time_millis}; +use tuliprox_core::utils::content_coding::DecodedHttpResponse; use tuliprox_parser::hls::origin_manifest::ParsedByteRange; pub(super) async fn fetch_segment_into_cache( @@ -22,101 +21,26 @@ pub(super) async fn fetch_segment_into_cache( fetch_segment_with_retries_into_cache(context, snapshot, policy).await } -pub(super) struct SegmentOriginAttemptGuard { - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -} - pub(super) async fn prepare_segment_origin_attempt( context: SegmentFetchContext, policy: SegmentFetchPolicy, + expected_generation: u64, ) -> Result { - let started_generation = start_segment_origin_work(&context).await; - let binding = - if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; - let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { - if binding.is_detached() { - finish_segment_origin_work(&context, started_generation).await; - touch_segment_origin_account_binding(&context, false).await; - return Err(SegmentFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); - } - let guard = match begin_hls_origin_account_io_bounded( - origin_io, - &context.session, - binding, - hls_object_body_deadline(policy.origin_segment_timeout_ms), - ) - .await - { - Ok(guard) => guard, - Err(err) => { - finish_segment_origin_work(&context, started_generation).await; - touch_segment_origin_account_binding(&context, false).await; - return Err(SegmentFetchError::ProviderUnavailable(err)); - } - }; - Some((origin_io.clone(), guard)) - } else { - None - }; - Ok(SegmentOriginAttemptGuard { started_generation, provider_lease }) + prepare_media_origin_attempt( + context.origin_io.as_ref(), + &context.session, + expected_generation, + hls_object_body_deadline(policy.origin_segment_timeout_ms), + ) + .await + .map_err(SegmentFetchError::ProviderUnavailable) } pub(super) async fn finish_segment_origin_attempt( context: SegmentFetchContext, guard: SegmentOriginAttemptGuard, ) -> SegmentOriginWorkFinish { - finish_segment_origin_io(&context, guard.started_generation, guard.provider_lease).await -} - -async fn finish_segment_origin_io( - context: &SegmentFetchContext, - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -) -> SegmentOriginWorkFinish { - let origin_work = finish_segment_origin_work(context, started_generation).await; - if let Some((origin_io, guard)) = provider_lease { - finish_hls_origin_account_io( - &origin_io, - &context.session, - guard, - origin_work.generation_valid && origin_work.refresh_reservation, - ) - .await; - touch_segment_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) - .await; - } - origin_work -} - -async fn start_segment_origin_work(context: &SegmentFetchContext) -> Option { - context.origin_io.as_ref()?; - let mut session = context.session.write().await; - Some(session.start_origin_work()) -} - -async fn finish_segment_origin_work( - context: &SegmentFetchContext, - started_generation: Option, -) -> SegmentOriginWorkFinish { - let Some(started_generation) = started_generation else { - return SegmentOriginWorkFinish { generation_valid: true, refresh_reservation: false }; - }; - let mut session = context.session.write().await; - let generation_valid = session.finish_origin_work(started_generation); - let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); - SegmentOriginWorkFinish { generation_valid, refresh_reservation } -} - -async fn touch_segment_origin_account_binding(context: &SegmentFetchContext, reservation_refreshed: bool) { - let mut session = context.session.write().await; - if let Some(binding) = session.origin_account_binding.as_mut() { - let now_ms = current_time_millis(); - binding.last_origin_io_at_ms = Some(now_ms); - if reservation_refreshed { - binding.last_reservation_refresh_at_ms = Some(now_ms); - } - } + guard.finish(&context.session).await } #[allow(clippy::too_many_lines)] @@ -125,11 +49,8 @@ pub(super) async fn fetch_segment_with_retries_into_cache( snapshot: &SegmentFetchSnapshot, policy: &SegmentFetchPolicy, ) -> Result { - let headers = build_segment_origin_headers( - &context.headers, - &context.origin_provider_session_headers, - snapshot.fetch_ref.byte_range, - )?; + let provider_headers = context.session.read().await.origin_provider_session_headers.clone(); + let headers = build_segment_origin_headers(&context.headers, &provider_headers, snapshot.fetch_ref.byte_range)?; let target = HlsOriginResourceFetchTarget { kind: HlsResourceFetchKind::Segment, source: HlsResourceFetchSource::Normal, @@ -151,6 +72,7 @@ pub(super) async fn fetch_segment_with_retries_into_cache( let session = context.session.read().await; super::super::HlsLogIdentity::from_session(&session) }; + let expected_generation = snapshot.origin_work_generation; let context = context.clone(); let snapshot = snapshot.clone(); let policy_for_prepare = policy.clone(); @@ -164,7 +86,7 @@ pub(super) async fn fetch_segment_with_retries_into_cache( move |_attempt| { let context = prepare_context.clone(); let policy = policy_for_prepare.clone(); - async move { prepare_segment_origin_attempt(context, policy).await }.boxed() + async move { prepare_segment_origin_attempt(context, policy, expected_generation).await }.boxed() }, move |guard| { let context = cleanup_context.clone(); diff --git a/backend/hls/src/segment_fetcher/scheduler.rs b/backend/hls/src/segment_fetcher/scheduler.rs index 99cc90e4f..fbc88bd33 100644 --- a/backend/hls/src/segment_fetcher/scheduler.rs +++ b/backend/hls/src/segment_fetcher/scheduler.rs @@ -107,7 +107,7 @@ pub(super) enum SegmentRetryWake { } #[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum CapacityRetryAdmission { +pub(super) enum CapacityRetryAdmission { Ready, BindingExpired, LocalIoFailure, @@ -189,6 +189,8 @@ impl HlsSegmentWorkerPool { access_leases, metrics, availability_reevaluations, + #[cfg(any(test, feature = "test-support"))] + origin_prepare_barrier: std::sync::Mutex::new(None), } } @@ -202,6 +204,13 @@ impl HlsSegmentWorkerPool { pub fn metrics(&self) -> &Arc { &self.metrics } + #[cfg(any(test, feature = "test-support"))] + pub fn pause_next_origin_prepare_for_test(&self) -> Arc { + let barrier = Arc::new(super::HlsSegmentPrepareBarrier::default()); + *self.origin_prepare_barrier.lock().unwrap() = Some(Arc::clone(&barrier)); + barrier + } + pub async fn classify_backpressure(&self, session: &HlsSessionHandle) -> HlsBackpressureState { let session = session.read().await; self.classify_backpressure_for_session(&session) @@ -217,6 +226,48 @@ impl HlsSegmentWorkerPool { } pub async fn wake_scheduler(self: &Arc, context: SegmentFetchContext, now_ms: u64) { + let (proxy_id, has_work, queued_generation) = { + let session = context.session.read().await; + ( + session.proxy_session_id.clone(), + !session.is_gc_marked_for_removal() && !session.segment_prefetch_queue.is_empty(), + session.activity.origin_work_generation, + ) + }; + #[cfg(any(test, feature = "test-support"))] + let barrier = if has_work { self.origin_prepare_barrier.lock().unwrap().take() } else { None }; + #[cfg(any(test, feature = "test-support"))] + if let Some(barrier) = &barrier { + barrier.entered.notify_one(); + barrier.prepare.notified().await; + } + let has_access = + has_work && self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_id, now_ms); + let preparation_failed = has_access + && crate::origin_account_runtime::prepare_media_origin(context.origin_io.as_ref(), &context.session) + .await + .is_err(); + if has_access && !preparation_failed { + let mut session = context.session.write().await; + // Preparation can invalidate the queue while renewing an account. + // Replan the current qualified window in the new work generation. + // Account handoff clears unqualified fetch refs and keeps this gate + // closed until its replacement manifest has committed. + if session.origin_handoff_ready_after(queued_generation) + && session.origin_account_binding.as_ref().is_none_or(crate::HlsOriginAccountBinding::is_active) + { + let backpressure = self.classify_backpressure_for_session(&session); + session.queue_manifest_fetch_candidates(now_ms, backpressure.allows_prefetch()); + } + } + #[cfg(any(test, feature = "test-support"))] + if let Some(barrier) = &barrier { + barrier.prepared.notify_one(); + barrier.proceed.notified().await; + } + if preparation_failed { + return; + } loop { let runtime = self.runtime.load_full(); let Ok(permit) = Arc::clone(&runtime.global_semaphore).try_acquire_owned() else { @@ -392,7 +443,9 @@ impl HlsSegmentWorkerPool { self.schedule_segment_retry(context, snapshot, retry); return; } - if completion.generation_valid { + if completion.generation_valid + || context.session.read().await.origin_handoff_ready_after(snapshot.origin_work_generation) + { self.schedule_wake(context, finished_at_ms); } } @@ -463,7 +516,7 @@ impl HlsSegmentWorkerPool { } } -async fn wait_for_capacity_retry_admission( +pub(super) async fn wait_for_capacity_retry_admission( context: &SegmentFetchContext, snapshot: &SegmentFetchSnapshot, mut revision: HlsCacheCapacityRevision, @@ -486,7 +539,9 @@ async fn wait_for_capacity_retry_admission( if !binding_current { return CapacityRetryAdmission::BindingExpired; } - match context.segment_cache.ensure_projected_write_capacity(&snapshot.cache_key, projected_write_bytes).await { + let admission = + context.segment_cache.ensure_revision_peak_capacity(&snapshot.cache_key, projected_write_bytes).await; + match admission { Ok(()) => return CapacityRetryAdmission::Ready, Err(error) => { let Some(capacity) = super::super::cache::hls_cache_capacity_from_io(&error) else { diff --git a/backend/hls/src/segment_fetcher/startup_fill.rs b/backend/hls/src/segment_fetcher/startup_fill.rs index 3d48ab8f1..f83c8a0d9 100644 --- a/backend/hls/src/segment_fetcher/startup_fill.rs +++ b/backend/hls/src/segment_fetcher/startup_fill.rs @@ -41,7 +41,7 @@ pub(super) struct HlsObservedFill { pub(super) budget: Arc, pub(super) deadline: Instant, pub(super) repair_permit: Option, - pub(super) additional_reservations: Vec, + pub(super) reservation: Arc, } impl Drop for HlsObservedFill { @@ -56,8 +56,16 @@ impl HlsObservedFill { .checked_add(u64::try_from(chunk.len()).map_err(std::io::Error::other)?) .ok_or_else(|| std::io::Error::other("origin body length overflow"))?; if self.bytes > self.limit { - self.extend_disk_reservation().await?; + self.extend_diskreservation().await?; } + self.context + .segment_cache + .ensure_write_budget_available( + &self.revision.revision().key, + &self.reservation, + self.bytes.saturating_mul(revision_peak_factor(self.file.is_some())), + ) + .await?; if let Some(file) = &mut self.file { file.write_all(chunk).await?; } @@ -104,7 +112,7 @@ impl HlsObservedFill { Ok(()) } - async fn extend_disk_reservation(&mut self) -> std::io::Result<()> { + async fn extend_diskreservation(&mut self) -> std::io::Result<()> { if self.expected.is_some() { return Err(std::io::Error::other("origin body exceeds decoded content length")); } @@ -114,12 +122,10 @@ impl HlsObservedFill { .checked_sub(self.limit) .and_then(|bytes| bytes.checked_mul(factor)) .ok_or_else(|| std::io::Error::other("revision disk reservation overflow"))?; - let mut reservation = - self.context.segment_cache.reserve_revision_peak(&self.revision.revision().key, additional_peak).await?; - if self.file.is_some() { - reservation.mark_unaccounted_writes(); - } - self.additional_reservations.push(reservation); + self.context + .segment_cache + .grow_revision_peak(&self.revision.revision().key, &self.reservation, additional_peak) + .await?; self.limit = next_limit; Ok(()) } @@ -129,10 +135,10 @@ impl HlsObservedFill { if self.bytes == 0 || self.expected.is_some_and(|length| length != self.bytes) { return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "incomplete origin segment body")); } - if let Some(file) = self.file.take() { - let mut file = file; + if let Some(file) = &mut self.file { file.flush().await?; - drop(file); + self.file.take(); + self.context.segment_cache.account_reserved_spool(&self.path, self.bytes, &self.reservation)?; self.revision.revision().complete(CachedSegmentMetadata { path: self.path.clone(), size: self.bytes }); self.revision.revision().replay.lock_unpoisoned().take(); } @@ -141,6 +147,7 @@ impl HlsObservedFill { } } +#[allow(clippy::too_many_lines, reason = "Keeps one startup write transaction and its failure closure together.")] pub(super) async fn commit_startup_response( context: &SegmentFetchContext, snapshot: &SegmentFetchSnapshot, @@ -149,6 +156,131 @@ pub(super) async fn commit_startup_response( mut repair_context: HlsSegmentRepairObjectContext, ) -> Result { use futures::StreamExt; + prepare_startup_repair_lease(context, &mut repair_context).await; + let expected = reliable_decoded_content_length(&response); + if response.status != axum::http::StatusCode::OK { + return Err(SegmentFetchError::cache_body(&std::io::Error::other( + "startup requires a complete HTTP 200 response", + ))); + } + let prepared = prepare_startup_fill(context, snapshot, expected, deadline).await?; + let PreparedStartupFill { revision, observed, reservation, raw } = prepared; + let mut transaction = + super::startup_write::StartupWrite::new(context, snapshot, Arc::clone(&reservation), revision.clone()); + transaction.track_fill(&observed); + let result = async { + let chunks_owner = Arc::clone(&observed); + let finish_owner = Arc::clone(&observed); + let stream = tokio_util::io::ReaderStream::new(response.body) + .then(move |chunk| { + let observed = Arc::clone(&chunks_owner); + async move { + let chunk = chunk?; + observed.lock().await.observe(&chunk).await?; + Ok::<_, std::io::Error>(chunk) + } + }) + .chain(futures::stream::once(async move { + finish_owner.lock().await.finish().await?; + Ok::<_, std::io::Error>(bytes::Bytes::new()) + })); + let reader = Box::pin(tokio_util::io::StreamReader::new(stream.boxed())); + #[cfg(test)] + context + .segment_cache + .startup_write_checkpoint(crate::cache::StartupWritePhase::BeforeNormalCommit) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + let mut metadata = context + .segment_repair + .commit_origin_response_with_budget( + &context.segment_cache, + &snapshot.cache_key, + reader, + deadline, + repair_context.clone(), + Some(Arc::clone(&reservation)), + ) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + #[cfg(test)] + context + .segment_cache + .startup_write_checkpoint(crate::cache::StartupWritePhase::AfterNormalCommit) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + if !raw { + context + .segment_repair + .repair_ready_cache_hit_with_budget( + &context.segment_cache, + &snapshot.cache_key, + repair_context, + Some(Arc::clone(&reservation)), + ) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + let latest = context + .segment_cache + .metadata(&snapshot.cache_key) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))? + .ok_or_else(|| { + SegmentFetchError::cache_body(&std::io::Error::other("processed cache object disappeared")) + })?; + metadata = latest; + let processed = context + .segment_cache + .publish_processed_revision_with_budget( + &revision.revision().key, + &metadata.path, + deadline, + Some(Arc::clone(&reservation)), + ) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + revision.revision().complete(processed); + } + #[cfg(test)] + context + .segment_cache + .startup_write_checkpoint(crate::cache::StartupWritePhase::AfterPublication) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + Ok(metadata) + } + .await; + if result.is_ok() { + transaction.finish(); + } else { + transaction.abort().await; + } + // A new origin attempt needs its entire peak again, even when the failed + // operation was a smaller revision file or a budget extension. + let peak = { + let fill = observed.lock().await; + let retry_bound = + if fill.bytes > fill.limit { fill.limit.saturating_mul(2).max(fill.bytes) } else { fill.limit }; + retry_bound.saturating_mul(revision_peak_factor(raw)) + }; + drop(observed); + result.map_err(|error| match error { + SegmentFetchError::LocalCacheCapacity { required_session_bytes, required_global_bytes, revision, .. } => { + SegmentFetchError::LocalCacheCapacity { + required_session_bytes, + required_global_bytes, + projected_write_bytes: peak, + revision, + } + } + other => other, + }) +} + +async fn prepare_startup_repair_lease( + context: &SegmentFetchContext, + repair_context: &mut HlsSegmentRepairObjectContext, +) { if repair_context.hls_access_lease_id.is_none() { let leases = context.session.read().await.startup.as_ref().map(|startup| Arc::clone(&startup.access_leases)); if let Some(leases) = leases { @@ -161,65 +293,12 @@ pub(super) async fn commit_startup_response( if let Some(lease_id) = &repair_context.hls_access_lease_id { context.segment_repair.ensure_access_lease_window(lease_id.clone()).await; } - let expected = reliable_decoded_content_length(&response); - if response.status != axum::http::StatusCode::OK { - return Err(SegmentFetchError::cache_body(&std::io::Error::other( - "startup requires a complete HTTP 200 response", - ))); - } - let prepared = prepare_startup_fill(context, snapshot, expected, deadline).await?; - let PreparedStartupFill { revision, observed, _reservation, raw } = prepared; - let chunks_owner = Arc::clone(&observed); - let finish_owner = Arc::clone(&observed); - let stream = tokio_util::io::ReaderStream::new(response.body) - .then(move |chunk| { - let observed = Arc::clone(&chunks_owner); - async move { - let chunk = chunk?; - observed.lock().await.observe(&chunk).await?; - Ok::<_, std::io::Error>(chunk) - } - }) - .chain(futures::stream::once(async move { - finish_owner.lock().await.finish().await?; - Ok::<_, std::io::Error>(bytes::Bytes::new()) - })); - let reader = Box::pin(tokio_util::io::StreamReader::new(stream.boxed())); - let mut metadata = context - .segment_repair - .commit_origin_response(&context.segment_cache, &snapshot.cache_key, reader, deadline, repair_context.clone()) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; - if !raw { - context - .segment_repair - .repair_ready_cache_hit(&context.segment_cache, &snapshot.cache_key, repair_context) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; - let latest = context - .segment_cache - .metadata(&snapshot.cache_key) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))? - .ok_or_else(|| { - SegmentFetchError::cache_body(&std::io::Error::other("processed cache object disappeared")) - })?; - metadata = latest; - let processed = context - .segment_cache - .publish_processed_revision(&revision.revision().key, &metadata.path, deadline) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; - revision.revision().complete(processed); - } - drop(observed); - Ok(metadata) } pub(super) struct PreparedStartupFill { pub(super) revision: super::super::SegmentRevisionGuard, pub(super) observed: Arc>, - pub(super) _reservation: super::super::HlsRevisionDiskReservation, + pub(super) reservation: Arc, pub(super) raw: bool, } @@ -238,6 +317,7 @@ pub(super) async fn prepare_startup_fill( } } +#[allow(clippy::too_many_lines)] async fn try_prepare_startup_fill( context: &SegmentFetchContext, snapshot: &SegmentFetchSnapshot, @@ -283,7 +363,8 @@ async fn try_prepare_startup_fill( let peak = body_bound .checked_mul(revision_peak_factor(raw)) .ok_or_else(|| SegmentFetchError::cache_body(&std::io::Error::other("revision peak overflow")))?; - let mut reservation = match context.segment_cache.reserve_revision_peak(&revision.revision().key, peak).await { + let mut reservation = match context.segment_cache.reserve_revision_peak_reclaiming(&snapshot.cache_key, peak).await + { Ok(reservation) => reservation, Err(_) if raw && decides_policy && fallback_to_first_ready(&context.session).await => { debug!("HLS startup fallback: reason=disk_capacity effective_mode=first_ready"); @@ -291,24 +372,44 @@ async fn try_prepare_startup_fill( } Err(error) => return Err(SegmentFetchError::cache_body(&error)), }; - let path = context.segment_cache.object_path(&revision.revision().key); + let path = reservation.object_path(&revision.revision().key); + if raw { + reservation.mark_unaccounted_writes(); + } + let reservation = Arc::new(reservation); + let mut preparation = + super::startup_write::StartupWrite::new(context, snapshot, Arc::clone(&reservation), revision.clone()); let file = if raw { - let (file, pin) = context + let (file, pin) = match context .segment_cache - .create_revision_spool(&revision.revision().key) + .create_revision_spool_with_budget(&revision.revision().key, Some(&reservation)) .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; + { + Ok(spool) => spool, + Err(error) => { + preparation.abort().await; + return Err(SegmentFetchError::cache_body(&error)); + } + }; *revision.revision().file_pin.lock_unpoisoned() = Some(pin); *revision.revision().replay.lock_unpoisoned() = replay.take(); - reservation.mark_unaccounted_writes(); Some(file) } else { - let pin = - context.segment_cache.pin_revision_file(&path).map_err(|error| SegmentFetchError::cache_body(&error))?; + let pin = match context.segment_cache.pin_revision_file(&path) { + Ok(pin) => pin, + Err(error) => { + preparation.abort().await; + return Err(SegmentFetchError::cache_body(&error)); + } + }; *revision.revision().file_pin.lock_unpoisoned() = Some(pin); None }; - bind_startup_revision(context, snapshot, &revision).await?; + if let Err(error) = bind_startup_revision(context, snapshot, &revision).await { + preparation.abort().await; + return Err(error); + } + preparation.finish(); debug!("HLS startup fill: mode={mode} proxy_seq={} representation={kind:?}", snapshot.proxy_seq); let observed = Arc::new(tokio::sync::Mutex::new(HlsObservedFill { revision: revision.clone(), @@ -323,9 +424,9 @@ async fn try_prepare_startup_fill( budget, deadline, repair_permit: None, - additional_reservations: Vec::new(), + reservation: Arc::clone(&reservation), })); - Ok(Some(PreparedStartupFill { revision, observed, _reservation: reservation, raw })) + Ok(Some(PreparedStartupFill { revision, observed, reservation, raw })) } async fn bind_startup_revision( diff --git a/backend/hls/src/segment_fetcher/startup_write.rs b/backend/hls/src/segment_fetcher/startup_write.rs new file mode 100644 index 000000000..8f9debf9e --- /dev/null +++ b/backend/hls/src/segment_fetcher/startup_write.rs @@ -0,0 +1,105 @@ +//! Failure/cancellation closure for one startup write, keyed by its peak owner. +use super::{segment_fetch_attempt_matches, SegmentFetchContext, SegmentFetchSnapshot}; +use crate::{sync_ext::MutexExt, HlsRevisionDiskReservation, SegmentRevisionGuard, SegmentRevisionState}; +use std::sync::Arc; + +/// Rolls back only this attempt and keeps immutable completed revisions alive. +pub(super) struct StartupWrite { + context: SegmentFetchContext, + snapshot: SegmentFetchSnapshot, + budget: Arc, + revision: SegmentRevisionGuard, + armed: bool, + observed: Option>>, +} + +impl StartupWrite { + pub(super) fn new( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + budget: Arc, + revision: SegmentRevisionGuard, + ) -> Self { + Self { context: context.clone(), snapshot: snapshot.clone(), budget, revision, armed: true, observed: None } + } + + pub(super) fn track_fill(&mut self, fill: &Arc>) { + self.observed = Some(Arc::clone(fill)); + } + + pub(super) fn finish(&mut self) { self.armed = false; } + + pub(super) async fn abort(&mut self) { + cleanup(&self.context, &self.snapshot, &self.budget, &self.revision, self.observed.as_ref()).await; + self.armed = false; + } +} + +impl Drop for StartupWrite { + fn drop(&mut self) { + if !self.armed { + return; + } + let context = self.context.clone(); + let snapshot = self.snapshot.clone(); + let budget = Arc::clone(&self.budget); + let revision = self.revision.clone(); + let observed = self.observed.clone(); + revision.revision().fail(); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(async move { + cleanup(&context, &snapshot, &budget, &revision, observed.as_ref()).await; + let mut session = context.session.write().await; + session.active_segment_fetches = session.active_segment_fetches.saturating_sub(1); + if session + .segments + .get(&snapshot.proxy_seq) + .is_some_and(|entry| segment_fetch_attempt_matches(entry, &snapshot)) + { + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = crate::SegmentCacheStatus::Discovered; + } + if let Some(notifier) = session.segment_fetch_notifiers.remove(&snapshot.proxy_seq) { + notifier.notify_waiters(); + } + } + }); + } + } +} + +async fn cleanup( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + budget: &HlsRevisionDiskReservation, + revision: &SegmentRevisionGuard, + observed: Option<&Arc>>, +) { + if let Some(observed) = observed { + use tokio::io::AsyncWriteExt; + let mut fill = observed.lock().await; + if let Some(mut file) = fill.file.take() { + let _ = file.flush().await; + } + } + revision.revision().fail(); + if let Err(error) = context.segment_cache.delete_write_owned(&snapshot.cache_key, budget).await { + log::warn!( + "HLS startup partial commit cleanup deferred: proxy_seq={} error_kind={:?}", + snapshot.proxy_seq, + error.kind() + ); + } + // Complete revisions have immutable, independently pinned bytes. Failed + // spools are unique to this revision and cannot be confused with a retry. + if matches!(revision.revision().state(), SegmentRevisionState::Failed) { + revision.revision().file_pin.lock_unpoisoned().take(); + if let Err(error) = context.segment_cache.delete_write_owned(&revision.revision().key, budget).await { + log::warn!( + "HLS startup failed revision cleanup deferred: proxy_seq={} error_kind={:?}", + snapshot.proxy_seq, + error.kind() + ); + } + } +} diff --git a/backend/hls/src/segment_fetcher/tests.rs b/backend/hls/src/segment_fetcher/tests.rs index e3fc02b48..dea31ae05 100644 --- a/backend/hls/src/segment_fetcher/tests.rs +++ b/backend/hls/src/segment_fetcher/tests.rs @@ -12,6 +12,7 @@ mod protocol; mod query; mod retry; mod startup; +mod startup_capacity; mod support; use self::support::{ diff --git a/backend/hls/src/segment_fetcher/tests/behavior.rs b/backend/hls/src/segment_fetcher/tests/behavior.rs index f77768134..68caa5586 100644 --- a/backend/hls/src/segment_fetcher/tests/behavior.rs +++ b/backend/hls/src/segment_fetcher/tests/behavior.rs @@ -428,7 +428,7 @@ async fn same_origin_sequence_resource_conflict_preserves_in_flight_binding() { let (committed_key, bytes) = committed_segment(&context, 1).await; assert_eq!(committed_key, cache_key); assert_eq!(bytes, b"controlled-segment"); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } #[tokio::test] @@ -495,7 +495,7 @@ async fn segment_decoder_failure_exhausts_attempt_budget_without_cache_or_temp_f let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } #[tokio::test] @@ -662,7 +662,7 @@ async fn unknown_body_reserves_more_disk_before_growing_its_raw_spool() -> std:: let mut observed = prepared.observed.lock().await; observed.limit = 8; observed.observe(b"segment-body").await?; - assert_eq!(observed.additional_reservations.len(), 1); + assert_eq!(observed.limit, 16); observed.finish().await?; let metadata = prepared.revision.revision().wait_complete(tokio::time::Instant::now() + Duration::from_secs(1)).await?; diff --git a/backend/hls/src/segment_fetcher/tests/lifecycle.rs b/backend/hls/src/segment_fetcher/tests/lifecycle.rs index 6f3ddbe60..018b0e006 100644 --- a/backend/hls/src/segment_fetcher/tests/lifecycle.rs +++ b/backend/hls/src/segment_fetcher/tests/lifecycle.rs @@ -2,10 +2,253 @@ use super::{ encrypted_fetch_context, fetch_context, spawn_segment_server, spawn_sequence_response_server, take_queued_segment_fetch_candidate, SegmentFetchPolicy, SegmentFetchPriority, TestOriginResponse, }; -use crate::{HlsSegmentFile, SegmentCacheStatus, TransientResourceKind}; +use crate::{ + api::{ + HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount, + LiveHlsOriginEntry, + }, + HlsSegmentFile, SegmentCacheStatus, TransientResourceKind, +}; use std::time::Duration; use tuliprox_core::utils::current_time_millis; +#[derive(Debug)] +struct InvalidatingPrepare { + first: std::sync::atomic::AtomicBool, + handoff: bool, +} + +impl HlsOriginAccountRuntime for InvalidatingPrepare { + fn prepare<'a>( + &'a self, + session: &'a crate::HlsSessionHandle, + _work: HlsOriginAccountWork, + _class: crate::HlsOriginWorkClass, + ) -> std::pin::Pin< + Box< + dyn std::future::Future> + + Send + + 'a, + >, + > { + Box::pin(async move { + let mut s = session.write().await; + if self.first.swap(false, std::sync::atomic::Ordering::AcqRel) { + if self.handoff { + let binding = crate::HlsOriginAccountBinding::new( + "input".into(), + "new-account".into(), + &s.proxy_session_id, + current_time_millis(), + ); + s.handoff_origin_account(binding); + } else { + s.invalidate_queued_origin_work(); + } + } + Ok(HlsPreparedOriginAccount { + entry: LiveHlsOriginEntry::parse("http://unused.example/index.m3u8").unwrap(), + snapshot: HlsOriginAccountSnapshot::from_session(&s), + }) + }) + } +} + +#[tokio::test] +async fn scheduler_replans_valid_manifest_work_after_prepare_invalidates_generation() { + use std::sync::{atomic::AtomicBool, Arc}; + use tuliprox_core::model::{Config, PlaybackKind}; + use tuliprox_session::ConnectionKind; + for handoff in [false, true] { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy::default(); + let (worker, mut context, _) = fetch_context(&server, &temp_dir, &policy).await; + super::support::grant_usable_worker_access_lease_at( + &worker, + &context.session.read().await.proxy_session_id, + current_time_millis(), + ) + .await; + let original = context.session.read().await.segment_prefetch_queue.proxy_seqs(); + assert_eq!(original.len(), 3); + let runtime = Arc::new(InvalidatingPrepare { first: AtomicBool::new(true), handoff }); + context.origin_io = Some(crate::HlsOriginIoContext { + ctx: crate::HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:12345".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: Some(runtime.clone()), + }); + worker.wake_scheduler(context.clone(), current_time_millis()).await; + assert!(!runtime.first.load(std::sync::atomic::Ordering::Acquire), "account preparation actually ran"); + assert_eq!(context.session.read().await.activity.origin_work_generation, 1); + if handoff { + let s = context.session.read().await; + assert!(s.origin_account_manifest_pending.is_some()); + assert!(s.segment_prefetch_queue.is_empty()); + assert!(original.iter().all(|seq| s.segments[seq].origin_fetch_ref.is_none())); + assert!(server.requests.lock().await.is_empty(), "unqualified old-account resources stay invalid"); + } else { + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let s = context.session.read().await; + if original.iter().all(|seq| matches!(s.segments[seq].status, SegmentCacheStatus::Ready { .. })) { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("valid jobs survive invalidation in prepare without test-side requeue"); + assert_eq!(server.requests.lock().await.len(), original.len()); + } + } +} + +#[tokio::test] +async fn scheduler_replan_uses_current_backpressure_for_prefetch() { + use crate::HlsBackpressureState; + use std::sync::{atomic::AtomicBool, Arc}; + use tuliprox_core::model::{Config, PlaybackKind}; + use tuliprox_session::ConnectionKind; + + for expected_pressure in + [HlsBackpressureState::Degraded, HlsBackpressureState::Saturated, HlsBackpressureState::Normal] + { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy { max_global_segment_fetches: 2, ..SegmentFetchPolicy::default() }; + let (worker, mut context, _) = fetch_context(&server, &temp_dir, &policy).await; + super::support::grant_usable_worker_access_lease_at( + &worker, + &context.session.read().await.proxy_session_id, + current_time_millis(), + ) + .await; + let semaphore = Arc::clone(&worker.runtime.load().global_semaphore); + let mut held_permits = vec![semaphore.try_acquire().unwrap()]; + let previous_generation = { + let mut s = context.session.write().await; + s.invalidate_queued_origin_work(); + s.initial_prefetch_gap_segments = 1; + let pressure = worker.classify_backpressure_for_session(&s); + assert_eq!(pressure, HlsBackpressureState::Degraded); + let report = s.queue_manifest_fetch_candidates(current_time_millis(), pressure.allows_prefetch()); + assert_eq!(report.render_window_queued, 2); + assert_eq!(report.prefetch_queued, 0); + assert_eq!(report.prefetch_skipped, 1); + s.activity.origin_work_generation + }; + let runtime = Arc::new(InvalidatingPrepare { first: AtomicBool::new(true), handoff: false }); + context.origin_io = Some(crate::HlsOriginIoContext { + ctx: crate::HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:12345".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: Some(runtime.clone()), + }); + let barrier = worker.pause_next_origin_prepare_for_test(); + let scheduler_worker = Arc::clone(&worker); + let scheduler_context = context.clone(); + let task = tokio::spawn(async move { + scheduler_worker.wake_scheduler(scheduler_context, current_time_millis()).await; + }); + tokio::time::timeout(Duration::from_secs(3), barrier.entered.notified()).await.unwrap(); + // Pressure can change after the manifest commit. Replanning must use its + // current classification, including recovery back to Normal. + match expected_pressure { + HlsBackpressureState::Normal => held_permits.clear(), + HlsBackpressureState::Degraded => {} + HlsBackpressureState::Saturated => held_permits.push(semaphore.try_acquire().unwrap()), + } + barrier.prepare.notify_one(); + tokio::time::timeout(Duration::from_secs(3), barrier.prepared.notified()).await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled(), "observe the replan before workers consume its queue"); + + assert!(!runtime.first.load(std::sync::atomic::Ordering::Acquire), "account preparation actually ran"); + let s = context.session.read().await; + assert_eq!(s.activity.origin_work_generation, previous_generation + 1); + let pressure = worker.classify_backpressure_for_session(&s); + assert_eq!(pressure, expected_pressure); + for seq in [1, 2] { + assert!(s.segment_prefetch_queue.contains(seq), "{pressure:?}: restore render-window work"); + assert!(matches!( + s.segments[&seq].status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, .. } + )); + } + let prefetch_queued = s.segment_prefetch_queue.contains(3); + eprintln!( + "replan pressure={pressure:?} generation={} render_window=[1, 2] prefetch_queued={prefetch_queued}", + s.activity.origin_work_generation + ); + assert_eq!( + prefetch_queued, + pressure.allows_prefetch(), + "{pressure:?}: replan must respect the current prefetch policy" + ); + assert_eq!( + matches!( + s.segments[&3].status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, .. } + ), + pressure.allows_prefetch() + ); + assert_eq!(s.segment_prefetch_queue.len(), if pressure.allows_prefetch() { 3 } else { 2 }); + assert!(server.requests.lock().await.is_empty(), "the barrier precedes origin segment work"); + } +} + +#[tokio::test] +async fn account_handoff_during_segment_prepare_blocks_old_origin_request() { + use crate::{origin_attempt::tests::bound_origin_io, HlsOriginAccountIoLease, HlsOriginResourceFetchError}; + use std::sync::{atomic::Ordering, Arc}; + + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let (worker, mut context, _) = fetch_context(&server, &temp_dir, &policy).await; + let (io, replacement) = bound_origin_io(&context.session).await; + context.origin_io = Some(io); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.expect("segment snapshot"); + let replacement_lease = Arc::new(HlsOriginAccountIoLease::active_for_test(&replacement, 1)); + let read_blocker = context.session.read().await; + let mut fetch = Box::pin(super::super::fetch_segment_with_retries_into_cache(&context, &snapshot, &policy)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + let mut handoff = Box::pin(context.session.write()); + assert!(futures::poll!(handoff.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + { + let mut session = handoff.await; + session.handoff_origin_account(replacement); + session.origin_account_io_lease = Some(Arc::clone(&replacement_lease)); + } + + let result = fetch.await; + assert!(server.requests.lock().await.is_empty(), "stale segment URL must never reach the origin"); + assert!(matches!(result, Err(HlsOriginResourceFetchError::ProviderUnavailable(_)))); + let session = context.session.read().await; + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert_eq!(replacement_lease.active_io_count.load(Ordering::Acquire), 1); +} + #[tokio::test] async fn popped_candidate_with_expired_fetch_binding_returns_to_discovered() { let server = spawn_sequence_response_server(vec![TestOriginResponse { diff --git a/backend/hls/src/segment_fetcher/tests/persistence.rs b/backend/hls/src/segment_fetcher/tests/persistence.rs index 1da9dae2a..ebf3debc3 100644 --- a/backend/hls/src/segment_fetcher/tests/persistence.rs +++ b/backend/hls/src/segment_fetcher/tests/persistence.rs @@ -88,5 +88,5 @@ async fn segment_decoder_failure_retries_then_commits_and_cleans_temp_file() { assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); drop(requests); assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } diff --git a/backend/hls/src/segment_fetcher/tests/retry.rs b/backend/hls/src/segment_fetcher/tests/retry.rs index 118135991..9b345aebd 100644 --- a/backend/hls/src/segment_fetcher/tests/retry.rs +++ b/backend/hls/src/segment_fetcher/tests/retry.rs @@ -399,7 +399,7 @@ async fn segment_decoded_object_limit_is_authoritative_and_non_retryable() { let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } #[tokio::test] diff --git a/backend/hls/src/segment_fetcher/tests/startup_capacity.rs b/backend/hls/src/segment_fetcher/tests/startup_capacity.rs new file mode 100644 index 000000000..9ca91015b --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/startup_capacity.rs @@ -0,0 +1,394 @@ +use super::*; +use crate::{cache::StartupWritePhase, SegmentCacheStatus}; +use std::{fmt::Write as _, sync::Arc, time::Duration}; +use tuliprox_core::utils::current_time_millis; + +#[tokio::test] +async fn startup_failure_and_cancellation_close_each_commit_transition() -> std::io::Result<()> { + for phase in [ + StartupWritePhase::BeforeNormalCommit, + StartupWritePhase::AfterNormalCommit, + StartupWritePhase::AfterPublication, + ] { + for cancel in [false, true] { + startup_transition_round(phase, cancel).await?; + } + } + Ok(()) +} + +async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std::io::Result<()> { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + let (entered, resume) = context.segment_cache.pause_startup_write(phase); + let task_context = context.clone(); + let task_snapshot = snapshot.clone(); + let task_policy = policy.clone(); + let task = tokio::spawn(async move { + super::super::fetch_segment_into_cache(&task_context, &task_snapshot, &task_policy).await + }); + tokio::time::timeout(Duration::from_secs(5), entered).await.unwrap().unwrap(); + let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&1].clone(); + if cancel { + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + drop(resume); + } else { + drop(resume); // A closed checkpoint injects a local I/O failure. + let result = task.await.unwrap(); + assert!(result.is_err()); + let mut session = context.session.write().await; + worker.apply_segment_fetch_result(&mut session, &snapshot, &policy, result, 11); + } + let id = snapshot.cache_key.proxy_session_id(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if context.segment_cache.remaining_revision_bytes(id) == 0 + && context.session.read().await.active_segment_fetches == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("bounded cleanup completes without GC or foreign mutations"); + assert!(context.segment_cache.metadata(&snapshot.cache_key).await?.is_none()); + assert_eq!(revision.revision().is_complete(), phase == StartupWritePhase::AfterPublication); + let bytes = context.segment_cache.capacity_usage(id).await?.session_bytes; + assert_eq!(bytes, if phase == StartupWritePhase::AfterPublication { 10 } else { 0 }); + assert_eq!(context.segment_cache.revision_disk_bytes(), context.segment_cache.scan_revision_disk_bytes().await?); + assert_eq!(temp_cache_files(directory.path()).len(), 0); + if phase == StartupWritePhase::AfterPublication { + for _ in 0..2 { + let metadata = revision.revision().wait_complete(tokio::time::Instant::now()).await?; + assert_eq!(tokio::fs::read(metadata.path).await?, b"body:/1.ts"); + } + } + Ok(()) +} + +#[tokio::test] +async fn capacity_retry_requires_the_complete_peak_and_only_reservation_release() -> std::io::Result<()> { + use super::super::scheduler::{wait_for_capacity_retry_admission, CapacityRetryAdmission}; + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + context.segment_cache.update_cache_limits(100, 100); + let held = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 70).await?; + let small = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 10).await?; + let sufficient = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 10).await?; + let error = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 30).await.err().unwrap(); + let revision = crate::cache::hls_cache_capacity_from_io(&error).unwrap().revision().clone(); + context.session.write().await.segments.get_mut(&1).unwrap().status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 11 }; + let waiting = wait_for_capacity_retry_admission(&context, &snapshot, revision, 30); + tokio::pin!(waiting); + assert!(futures::poll!(waiting.as_mut()).is_pending()); + drop(small); + assert!(futures::poll!(waiting.as_mut()).is_pending(), "20 free bytes cannot admit a fresh 30-byte peak"); + assert!(server.requests.lock().await.is_empty()); + drop(sufficient); + assert_eq!(tokio::time::timeout(Duration::from_secs(2), waiting).await.unwrap(), CapacityRetryAdmission::Ready); + assert!(server.requests.lock().await.is_empty()); + drop(held); + Ok(()) +} + +#[tokio::test] +async fn thousand_live_windows_keep_one_manager_two_leases_and_bounded_revision_disk() -> std::io::Result<()> { + tokio::time::timeout(Duration::from_secs(60), live_window_soak()) + .await + .expect("1000-window soak makes bounded progress") +} + +#[allow(clippy::too_many_lines)] +async fn live_window_soak() -> std::io::Result<()> { + use crate::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey, + }; + use shared::model::{ByteSize, HlsCacheConfigDto, HlsStartupMode}; + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + cache_duration: shared::model::Secs::new(1), + cache_bytes: ByteSize::new("512 B"), + cache_bytes_per_session: ByteSize::new("256 B"), + max_concurrent_segment_fetches_per_session: 1, + ..Default::default() + }; + dto.startup.mode = HlsStartupMode::FirstReady; + let manager = HlsProxyManager::with_hls_cache_config(&tuliprox_core::model::HlsCacheConfig::from(&dto)); + let now = current_time_millis(); + let session_key = HlsSessionKey::new(1, "capacity-soak"); + let (session, _) = manager.get_or_create_session_with_outcome(session_key.clone(), b"secret", now).await; + let id = session.read().await.proxy_session_id.clone(); + let lease_ids = [HlsAccessLeaseId("soak-a".into()), HlsAccessLeaseId("soak-b".into())]; + for lease_id in &lease_ids { + let mut lease = HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("test", &lease_id.0), + id.clone(), + "test".into(), + "test-session".into(), + 1, + "capacity-soak".into(), + 1, + now, + 1_000_000, + ); + lease.state = HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(now + 1_000_000); + assert!(manager.access_leases().write().await.prepare_access_lease(lease)); + } + let (_, mut context, _) = fetch_context(&server, &directory, &SegmentFetchPolicy::default()).await; + context.session = Arc::clone(&session); + context.segment_cache = Arc::clone(manager.segment_cache()); + context.segment_repair = Arc::clone(manager.segment_repair()); + let worker = Arc::clone(manager.segment_worker_pool()); + let policy = worker.policy(); + let mut downloads = 0usize; + let mut injected_failures = 0usize; + let mut reclaimed = 0u64; + for head in 0u64..1000 { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + for seq in head..head + 6 { + writeln!(body, "#EXTINF:4,\n{}/{seq}.ts", server.base_url).unwrap(); + } + let keys = { + let mut guard = session.write().await; + guard.last_client_access_at_ms = now + head; + guard.apply_origin_manifest(&normal_manifest(&body)).unwrap(); + guard + .segments + .values() + .filter(|entry| { + entry.origin_fetch_ref.is_some() && !matches!(entry.status, SegmentCacheStatus::Ready { .. }) + }) + .map(|entry| entry.cache_key.clone()) + .collect::>() + }; + for key in keys { + let inject = head > 0 && head.is_multiple_of(100); + if inject { + let (entered, resume) = context.segment_cache.pause_startup_write(StartupWritePhase::AfterNormalCommit); + let fill = soak_fetch(&worker, &context, &policy, key.proxy_seq(), now + head); + tokio::pin!(fill); + tokio::select! { + reached = entered => reached.unwrap(), + result = &mut fill => panic!("fill bypassed injected checkpoint: {}", result.is_ok()), + } + drop(resume); + assert!(fill.await.is_err()); + injected_failures += 1; + session.write().await.segments.get_mut(&key.proxy_seq()).unwrap().status = + SegmentCacheStatus::Discovered; + } + soak_fetch(&worker, &context, &policy, key.proxy_seq(), now + head).await?; + downloads += 1; + let segment = crate::HlsSegmentFile { proxy_seq: key.proxy_seq(), extension: "ts".into() }; + for _ in 0..2 { + assert_eq!( + worker.demand_fetch_and_wait(context.clone(), &segment, now + head).await, + super::super::SegmentDemandFetchOutcome::Ready + ); + } + } + let snapshot = { + let mut guard = session.write().await; + guard.render_and_store_manifest(now + head).unwrap(); + let rendered = guard.last_rendered_manifest.as_ref().unwrap(); + crate::manifest_snapshot::derive_hls_lease_manifest_snapshot( + &crate::manifest_snapshot::HlsLeaseManifestSnapshotInput::NormalCacheTimeline { + session: &guard, + committed_body: &rendered.body, + materialized_body: &rendered.body, + stripped_tail_segments: 0, + }, + now + head, + ) + .unwrap() + .unwrap() + }; + for lease_id in &lease_ids { + let expected = manager.prepare_access_lease_manifest_publication(lease_id, &id, now + head).await.unwrap(); + assert!(manager + .commit_access_lease_manifest_publication(lease_id, &id, expected, snapshot.clone(), now + head) + .await + .is_committed()); + } + drop(snapshot); + let report = manager.run_garbage_collection_once(now + head + 60_000).await?; + reclaimed += report.revision_bytes_reclaimed; + let usage = context.segment_cache.capacity_usage(&id).await?; + assert!(usage.session_bytes <= 256 && usage.global_bytes <= 512); + assert_eq!(context.segment_cache.remaining_revision_bytes(&id), 0); + assert_eq!( + context.segment_cache.revision_disk_bytes(), + context.segment_cache.scan_revision_disk_bytes().await? + ); + if head.is_multiple_of(100) { + eprintln!("live-window={head} downloads={downloads} revision_gc_bytes={reclaimed} injected_failures={injected_failures}"); + } + } + for lease in &lease_ids { + manager.access_leases().write().await.remove_access_lease(lease); + } + for _ in 0..16 { + if manager.run_garbage_collection_once(now + 61_000).await?.revision_deletions_planned == 0 { + break; + } + } + let guard = session.read().await; + let startup = guard.startup.as_ref().unwrap(); + assert!(startup.store.retire_unpinned()?.is_empty()); + let retained = startup + .revisions + .values() + .filter_map(|owner| match owner.revision().state() { + crate::SegmentRevisionState::Complete(metadata) => Some(metadata.size), + crate::SegmentRevisionState::Pending | crate::SegmentRevisionState::Failed => None, + }) + .sum::(); + assert_eq!(context.segment_cache.scan_revision_disk_bytes().await?, retained); + assert_eq!(context.segment_cache.remaining_revision_bytes(&id), 0); + assert!(downloads >= 1000 && injected_failures >= 9 && reclaimed > 0); + drop(guard); + let (same_session, _) = manager.get_or_create_session_with_outcome(session_key, b"secret", now + 1001).await; + assert!(Arc::ptr_eq(&session, &same_session)); + assert_eq!(server.requests.lock().await.len(), downloads + injected_failures); + eprintln!( + "live-window=1000 downloads={downloads} revision_gc_bytes={reclaimed} injected_failures={injected_failures}" + ); + Ok(()) +} + +async fn soak_fetch( + worker: &Arc, + context: &SegmentFetchContext, + policy: &SegmentFetchPolicy, + seq: u64, + now: u64, +) -> std::io::Result<()> { + assert!(context.session.write().await.queue_segment_fetch_candidate(seq, SegmentFetchPriority::Demand, now)); + let snapshot = worker.next_fetch_snapshot(context, now, policy).await.unwrap(); + let result = super::super::fetch_segment_into_cache(context, &snapshot, policy).await; + let success = result.is_ok(); + let mut session = context.session.write().await; + worker.apply_segment_fetch_result(&mut session, &snapshot, policy, result, now); + if success { + Ok(()) + } else { + Err(std::io::Error::other("injected or unexpected fill failure")) + } +} + +#[tokio::test] +async fn processed_peak_release_resumes_a_real_deferred_download_without_gc() -> std::io::Result<()> { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, segment) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await; + clear_scheduled_prefetch(&context, &policy).await; + context.segment_cache.update_cache_limits(100, 100); + let key = context.session.read().await.segments[&1].cache_key.clone(); + let held = context.segment_cache.reserve_revision_peak(&key, 80).await?; + let demand = worker.demand_fetch_and_wait(context.clone(), &segment, current_time_millis()); + tokio::pin!(demand); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + tokio::select! { + outcome = &mut demand => panic!("premature demand completion: {outcome:?}"), + () = tokio::task::yield_now() => {}, + } + if matches!(context.session.read().await.segments[&1].status, SegmentCacheStatus::CapacityDeferred { .. }) { + break; + } + } + }) + .await + .unwrap(); + assert_eq!(server.requests.lock().await.len(), 1); + drop(held); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), demand).await.unwrap(), + super::super::SegmentDemandFetchOutcome::Ready + ); + assert_eq!(server.requests.lock().await.len(), 2); + assert_eq!(context.segment_cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(tokio::fs::read(context.segment_cache.object_path(&key)).await?, b"body:/1.ts"); + Ok(()) +} + +#[tokio::test] +async fn unknown_and_growing_bodies_abort_before_writing_unreserved_bytes() -> std::io::Result<()> { + use futures::StreamExt; + for mode in [shared::model::HlsStartupMode::FirstReady, shared::model::HlsStartupMode::Progressive] { + for expected in [None, Some(8)] { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, mode).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.startup.as_mut().unwrap().config.max_progressive_bytes_per_segment = + shared::model::Bytes::new(8); + context.segment_cache.update_cache_limits(45, 45); + context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let prepared = super::super::prepare_startup_fill(&context, &snapshot, expected, deadline) + .await + .map_err(|error| std::io::Error::other(format!("{error:?}")))?; + let mut transaction = super::super::startup_write::StartupWrite::new( + &context, + &snapshot, + Arc::clone(&prepared.reservation), + prepared.revision.clone(), + ); + transaction.track_fill(&prepared.observed); + let observed = Arc::clone(&prepared.observed); + let stream = futures::stream::iter([b"12345678".as_slice(), b"extra".as_slice()]).then(move |chunk| { + let observed = Arc::clone(&observed); + async move { + observed.lock().await.observe(chunk).await?; + Ok::<_, std::io::Error>(bytes::Bytes::copy_from_slice(chunk)) + } + }); + let reader = Box::pin(tokio_util::io::StreamReader::new(stream.boxed())); + let result = context + .segment_cache + .stage_temp_with_budget(&snapshot.cache_key, reader, deadline, Some(Arc::clone(&prepared.reservation))) + .await; + assert!(result.is_err()); + // The first chunk was covered; the rejected second chunk never + // reached either the normal staging file or the raw spool. + if prepared.raw { + assert_eq!(tokio::fs::metadata(&prepared.observed.lock().await.path).await?.len(), 8); + } + transaction.abort().await; + drop(transaction); + drop(prepared); + assert_eq!( + context.segment_cache.capacity_usage(snapshot.cache_key.proxy_session_id()).await?.session_bytes, + 0 + ); + assert_eq!(context.segment_cache.remaining_revision_bytes(snapshot.cache_key.proxy_session_id()), 0); + assert_eq!(temp_cache_files(directory.path()).len(), 0); + } + } + Ok(()) +} diff --git a/backend/hls/src/segment_repair.rs b/backend/hls/src/segment_repair.rs index 8ce7d9be3..d94da19ef 100644 --- a/backend/hls/src/segment_repair.rs +++ b/backend/hls/src/segment_repair.rs @@ -38,6 +38,7 @@ struct HlsSelectedRepairCandidate<'a> { runtime: Arc, candidate: HlsRepairWindowCandidateKey, prewarm_guard: Option<&'a HlsRepairPrewarmGuard>, + write_budget: Option>, } #[derive(Debug, Clone, Eq, PartialEq, Hash)] @@ -188,6 +189,8 @@ struct RepairRemuxDroppedStream { mod tests; mod diagnostics; +mod disk_output; +pub(crate) use disk_output::{run_reserved_remux, RepairDiskBudget}; mod manager; mod probe; mod processing; diff --git a/backend/hls/src/segment_repair/disk_output.rs b/backend/hls/src/segment_repair/disk_output.rs new file mode 100644 index 000000000..5814e37a6 --- /dev/null +++ b/backend/hls/src/segment_repair/disk_output.rs @@ -0,0 +1,137 @@ +//! Streams postprocessor output through admission before each filesystem write. +use super::{HlsPostProcessingDeadline, HlsSegmentCache, ProxySessionId, SegmentCacheKey, StagedCacheObject}; +use std::{io, sync::Arc}; +use tokio::{ + fs::File, + io::{AsyncReadExt, AsyncWriteExt}, + process::Command, + time::timeout, +}; + +/// Peak ownership and still-live staging bytes for streamed repair output. +#[derive(Clone, Copy)] +pub(crate) struct RepairDiskBudget<'a> { + pub(crate) cache: &'a HlsSegmentCache, + pub(crate) reservation: &'a Arc, + pub(crate) session: &'a ProxySessionId, + pub(crate) retained_staging_bytes: u64, +} + +async fn copy_reserved_output( + reader: &mut R, + file: &mut File, + budget: RepairDiskBudget<'_>, +) -> io::Result<()> { + let key = SegmentCacheKey::new(budget.session.clone(), 0, "ts"); + let mut size = 0u64; + let mut buffer = [0u8; 16 * 1024]; + loop { + let read = reader.read(&mut buffer).await?; + if read == 0 { + break; + } + size = size + .checked_add(u64::try_from(read).map_err(io::Error::other)?) + .ok_or_else(|| io::Error::other("postprocessor output overflow"))?; + budget + .cache + .ensure_write_budget_available(&key, budget.reservation, size.saturating_add(budget.retained_staging_bytes)) + .await?; + file.write_all(&buffer[..read]).await?; + } + file.flush().await?; + Ok(()) +} + +/// Runs an MPEG-TS writer with stdout backpressure so it cannot grow a disk +/// file past its peak reservation between two capacity checks. +pub(crate) async fn run_reserved_remux( + args: &[&str], + output: &StagedCacheObject, + budget: RepairDiskBudget<'_>, + deadline: &HlsPostProcessingDeadline, +) -> Result<(), String> { + let remaining = deadline.remaining().ok_or_else(|| "timeout".to_string())?; + timeout(remaining, async { + let writer = output.track_writer(File::create(&output.path).await?); + let mut writer = writer.lock().await; + let file = writer.as_mut().ok_or_else(|| io::Error::other("postprocessor output already closed"))?; + let mut command = Command::new("ffmpeg"); + let mut child = command + .args(args) + .arg("pipe:1") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn()?; + let mut stdout = child.stdout.take().ok_or_else(|| io::Error::other("postprocessor stdout unavailable"))?; + copy_reserved_output(&mut stdout, file, budget).await?; + writer.take(); + if !child.wait().await?.success() { + return Err(io::Error::other("postprocessor failed")); + } + Ok::<(), io::Error>(()) + }) + .await + .map_err(|_| "timeout".to_string())? + .map_err(|error| format!("reserved_postprocess_error:{:?}", error.kind())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn postprocessor_output_extends_its_peak_before_writing_or_aborts_without_overshoot() -> io::Result<()> { + for output_size in [100usize, 128] { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(120, 120); + let session = ProxySessionId("postprocess".into()); + let key = SegmentCacheKey::new(session.clone(), 0, "ts"); + let reservation = Arc::new(cache.reserve_revision_peak(&key, 24).await?); + let raw = cache + .stage_temp_with_budget( + &key, + &[0; 8][..], + tokio::time::Instant::now() + std::time::Duration::from_secs(2), + Some(reservation.clone()), + ) + .await?; + let path = raw.path.with_extension("ts.tmp.output"); + let mut output = cache.adopt_staged_file(path.clone(), 0)?.with_write_budget(Some(reservation.clone()))?; + let mut file = File::create(&path).await?; + let bytes = vec![7u8; output_size]; + let result = copy_reserved_output( + &mut bytes.as_slice(), + &mut file, + RepairDiskBudget { + cache: &cache, + reservation: &reservation, + session: &session, + retained_staging_bytes: raw.size, + }, + ) + .await; + drop(file); + output.size = tokio::fs::metadata(&path).await?.len(); + assert!(output.size + raw.size <= 120); + if output_size == 100 { + result?; + assert_eq!(output.size, 100); + assert!(cache.remaining_revision_bytes(&session) >= 108); + cache.remove_staged(raw).await?; + cache.commit_staged(&key, output).await?; + assert_eq!(cache.capacity_usage(&session).await?.session_bytes, 100); + } else { + assert!(result.is_err()); + assert_eq!(output.size, 0); + cache.remove_staged(raw).await?; + cache.remove_staged(output).await?; + } + drop(reservation); + assert_eq!(cache.remaining_revision_bytes(&session), 0); + } + Ok(()) + } +} diff --git a/backend/hls/src/segment_repair/manager.rs b/backend/hls/src/segment_repair/manager.rs index f541f30a2..5151d9670 100644 --- a/backend/hls/src/segment_repair/manager.rs +++ b/backend/hls/src/segment_repair/manager.rs @@ -1,11 +1,11 @@ use super::{ - ffmpeg_identity_version, probe::adopt_repair_output, registry::repair_object_metadata_key, - safe_hls_access_lease_id, safe_proxy_session_id, sha256_file, window::RepairStatus, CachedSegmentMetadata, - HlsAccessLeaseId, HlsCacheObjectKey, HlsCorruptSegmentWatchdogManager, HlsPostProcessingDeadline, - HlsRepairObjectMetadata, HlsRepairObjectMetadataKey, HlsRepairPrewarmGuard, HlsRepairRenderedObjectId, - HlsRepairWindowRegistry, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, - HlsSegmentRepairRuntime, HlsSegmentRepairStats, HlsSelectedRepairCandidate, ProxySessionId, RepairIdentity, - SegmentRepairMetadata, COMMAND_VERSION, REPAIR_METADATA_MAX_ENTRIES, REPAIR_OBJECT_METADATA_MAX_ENTRIES, + ffmpeg_identity_version, registry::repair_object_metadata_key, safe_hls_access_lease_id, safe_proxy_session_id, + sha256_file, window::RepairStatus, CachedSegmentMetadata, HlsAccessLeaseId, HlsCacheObjectKey, + HlsCorruptSegmentWatchdogManager, HlsPostProcessingDeadline, HlsRepairObjectMetadata, HlsRepairObjectMetadataKey, + HlsRepairPrewarmGuard, HlsRepairRenderedObjectId, HlsRepairWindowRegistry, HlsSegmentCache, + HlsSegmentRepairManager, HlsSegmentRepairObjectContext, HlsSegmentRepairRuntime, HlsSegmentRepairStats, + HlsSelectedRepairCandidate, ProxySessionId, RepairIdentity, SegmentRepairMetadata, COMMAND_VERSION, + REPAIR_METADATA_MAX_ENTRIES, REPAIR_OBJECT_METADATA_MAX_ENTRIES, }; use arc_swap::ArcSwap; use log::debug; @@ -16,7 +16,6 @@ use std::{ sync::Arc, }; use tokio::{ - fs, io::AsyncRead, sync::{Mutex, RwLock, Semaphore}, }; @@ -157,7 +156,13 @@ impl HlsSegmentRepairManager { let segment_cache = Arc::clone(&segment_cache); let guard = guard.clone(); tokio::spawn(async move { - let selection = HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: guard.as_ref() }; + let selection = HlsSelectedRepairCandidate { + mode, + runtime, + candidate, + prewarm_guard: guard.as_ref(), + write_budget: None, + }; if let Err(err) = manager.repair_ready_cache_hit_with_candidate(&segment_cache, &key, context, selection).await { @@ -210,7 +215,23 @@ impl HlsSegmentRepairManager { K: HlsCacheObjectKey, R: AsyncRead + Unpin, { - let raw = segment_cache.stage_temp_with_deadline(key, reader, deadline).await?; + self.commit_origin_response_with_budget(segment_cache, key, reader, deadline, context, None).await + } + + pub(crate) async fn commit_origin_response_with_budget( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + reader: R, + deadline: tokio::time::Instant, + context: HlsSegmentRepairObjectContext, + budget: Option>, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + let raw = segment_cache.stage_temp_with_budget(key, reader, deadline, budget).await?; let runtime = self.runtime.load_full(); if !runtime.postprocessing_enabled() { return segment_cache.commit_staged(key, raw).await; @@ -229,6 +250,16 @@ impl HlsSegmentRepairManager { where K: HlsCacheObjectKey, { + self.repair_ready_cache_hit_with_budget(segment_cache, key, context, None).await + } + + pub(crate) async fn repair_ready_cache_hit_with_budget( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + context: HlsSegmentRepairObjectContext, + write_budget: Option>, + ) -> io::Result> { let Some((mode, runtime, candidate)) = self.try_select_or_join_candidate(&context).await else { return Ok(None); }; @@ -236,7 +267,7 @@ impl HlsSegmentRepairManager { segment_cache, key, context, - HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: None }, + HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: None, write_budget }, ) .await } @@ -252,7 +283,7 @@ impl HlsSegmentRepairManager { where K: HlsCacheObjectKey, { - let HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard } = selection; + let HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard, write_budget } = selection; let Some(metadata) = segment_cache.metadata(key).await? else { return Ok(None); }; @@ -296,10 +327,19 @@ impl HlsSegmentRepairManager { Ok(None) } else { let deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); - if let Some(fixed_path) = self + if let Some(staged) = self .repair_file( - ¤t_metadata.path, - current_metadata.size, + super::processing::RepairFileSource { + cache: segment_cache, + path: ¤t_metadata.path, + size: current_metadata.size, + disk_budget: write_budget.as_ref().map(|reservation| super::RepairDiskBudget { + cache: segment_cache, + reservation, + session: &context.proxy_session_id, + retained_staging_bytes: 0, + }), + }, &identity, &context, runtime.clone(), @@ -317,11 +357,9 @@ impl HlsSegmentRepairManager { None => None, }; if !candidate_current || latest_hash.as_deref() != Some(current_hash.as_str()) { - let _ = fs::remove_file(&fixed_path).await; + segment_cache.remove_staged(staged).await?; Ok(None) } else { - let fixed_size = fs::metadata(&fixed_path).await?.len(); - let staged = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; let committed = segment_cache.commit_staged(key, staged).await?; self.record_metadata( identity.clone(), diff --git a/backend/hls/src/segment_repair/probe.rs b/backend/hls/src/segment_repair/probe.rs index 83eccb9df..b1baf9a01 100644 --- a/backend/hls/src/segment_repair/probe.rs +++ b/backend/hls/src/segment_repair/probe.rs @@ -1,9 +1,9 @@ use super::{ diagnostics::{decide_repair, SegmentProbeStreamType}, window::{RepairStatus, RepairVideoCodec}, - HlsSegmentCache, RepairRemuxDroppedStream, RepairRemuxStreamSelection, SegmentProbe, SegmentProbeStream, - StagedCacheObject, WarningCounters, + RepairRemuxDroppedStream, RepairRemuxStreamSelection, SegmentProbe, SegmentProbeStream, WarningCounters, }; +#[cfg(test)] use log::warn; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -12,7 +12,9 @@ use std::{ fmt, io, path::{Path, PathBuf}, }; -use tokio::{fs, fs::File, io::AsyncReadExt}; +#[cfg(test)] +use tokio::fs; +use tokio::{fs::File, io::AsyncReadExt}; use tuliprox_core::model::HlsSegmentRepairConfig; pub(super) fn parse_probe(json: &str, warnings: WarningCounters) -> Result { @@ -240,6 +242,7 @@ pub(super) fn repair_output_path(raw_path: &Path) -> PathBuf { raw_path.with_file_name(format!("{file_name}.repair.tmp.{suffix:016x}")) } +#[cfg(test)] pub(super) async fn cleanup_repair_output(path: &Path) { match fs::remove_file(path).await { Ok(()) => {} @@ -248,20 +251,6 @@ pub(super) async fn cleanup_repair_output(path: &Path) { } } -pub(super) async fn adopt_repair_output( - segment_cache: &HlsSegmentCache, - path: PathBuf, - size: u64, -) -> io::Result { - match segment_cache.adopt_staged_file(path.clone(), size) { - Ok(staged) => Ok(staged), - Err(error) => { - cleanup_repair_output(&path).await; - Err(error) - } - } -} - impl fmt::Display for RepairStatus { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -275,3 +264,18 @@ impl fmt::Display for RepairStatus { } } } + +#[cfg(test)] +pub(super) async fn adopt_repair_output( + segment_cache: &super::HlsSegmentCache, + path: PathBuf, + size: u64, +) -> io::Result { + match segment_cache.adopt_staged_file(path.clone(), size) { + Ok(staged) => Ok(staged), + Err(error) => { + cleanup_repair_output(&path).await; + Err(error) + } + } +} diff --git a/backend/hls/src/segment_repair/processing.rs b/backend/hls/src/segment_repair/processing.rs index 9106d4934..199c5c005 100644 --- a/backend/hls/src/segment_repair/processing.rs +++ b/backend/hls/src/segment_repair/processing.rs @@ -1,10 +1,7 @@ use super::{ diagnostics::{debug_repair_analysis, debug_repair_event, debug_repair_stream_dropped, decide_repair}, ffmpeg_identity_version, parse_ffmpeg_warnings, - probe::{ - adopt_repair_output, cleanup_repair_output, detect_video_codec, parse_probe, repair_output_path, - select_repair_remux_streams, validate_repair, - }, + probe::{detect_video_codec, parse_probe, repair_output_path, select_repair_remux_streams, validate_repair}, registry::repair_object_metadata_key, sha256_file, window::{RepairCandidateSelection, RepairStatus}, @@ -15,15 +12,19 @@ use super::{ }; use log::{debug, warn}; use shared::model::{HlsSegmentRepairExecutionPlan, HlsSegmentRepairMode}; -use std::{ - io, - path::{Path, PathBuf}, - sync::Arc, - time::Instant, -}; +use std::{io, path::Path, sync::Arc, time::Instant}; use tokio::{fs, process::Command, time::timeout}; +/// Files and peak ownership used for one postprocessing attempt. +pub(super) struct RepairFileSource<'a> { + pub(super) cache: &'a HlsSegmentCache, + pub(super) path: &'a Path, + pub(super) size: u64, + pub(super) disk_budget: Option>, +} + impl HlsSegmentRepairManager { + #[allow(clippy::too_many_lines)] pub(super) async fn process_staged_and_commit( &self, segment_cache: &HlsSegmentCache, @@ -91,14 +92,27 @@ impl HlsSegmentRepairManager { Ok(committed) } else { let raw_size = raw.size; - if let Some(fixed_path) = - self.repair_file(&raw.path, raw.size, &identity, &context, runtime.clone(), &deadline).await? + if let Some(fixed) = self + .repair_file( + RepairFileSource { + cache: segment_cache, + path: &raw.path, + size: raw.size, + disk_budget: raw.write_budget.as_ref().map(|reservation| super::RepairDiskBudget { + cache: segment_cache, + reservation, + session: &context.proxy_session_id, + retained_staging_bytes: raw.size, + }), + }, + &identity, + &context, + runtime.clone(), + &deadline, + ) + .await? { - let fixed_size = fs::metadata(&fixed_path).await?.len(); - if let Err(error) = segment_cache.remove_staged(raw.clone()).await { - warn!("HLS repair raw staging cleanup deferred: error_kind={:?}", error.kind()); - } - let fixed = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; + segment_cache.remove_staged(raw.clone()).await?; let committed = segment_cache.commit_staged(key, fixed).await?; self.record_metadata(identity.clone(), RepairStatus::Fixed, raw.size, committed.size, None).await; let committed_hash = sha256_file(&committed.path).await?; @@ -201,13 +215,13 @@ impl HlsSegmentRepairManager { #[allow(clippy::too_many_lines)] pub(super) async fn repair_file( &self, - raw_path: &Path, - raw_size: u64, + source: RepairFileSource<'_>, identity: &RepairIdentity, context: &HlsSegmentRepairObjectContext, runtime: Arc, deadline: &HlsPostProcessingDeadline, - ) -> io::Result> { + ) -> io::Result> { + let RepairFileSource { cache: segment_cache, path: raw_path, size: raw_size, disk_budget } = source; let _permit = match &runtime.semaphore { Some(semaphore) => { let Some(remaining) = deadline.remaining() else { @@ -297,11 +311,30 @@ impl HlsSegmentRepairManager { identity.repair_mode.as_log_value(), executed_level.as_log_value() ); - let remux = run_remux(raw_path, &fixed_path, executed_level, &stream_selection, deadline).await; + // A ready-cache prewarm has no startup owner. Give its output the same + // admission and cancellation guarantees as a downloaded staging file. + let owned_budget; + let disk_budget = if let Some(budget) = disk_budget { + budget + } else { + let key = crate::SegmentCacheKey::new(context.proxy_session_id.clone(), 0, "ts"); + owned_budget = Arc::new(segment_cache.reserve_revision_peak_reclaiming(&key, raw_size).await?); + super::RepairDiskBudget { + cache: segment_cache, + reservation: &owned_budget, + session: &context.proxy_session_id, + retained_staging_bytes: 0, + } + }; + let mut output = segment_cache + .adopt_staged_file(fixed_path.clone(), 0)? + .with_write_budget(Some(Arc::clone(disk_budget.reservation)))?; + let remux = + run_remux(raw_path, &fixed_path, executed_level, &stream_selection, deadline, disk_budget, &output).await; if let Err(reason) = remux { debug_repair_event(context, executed_level, "remux failed", Some(&reason)); let status = if reason == "timeout" { RepairStatus::Timeout } else { RepairStatus::RemuxFailed }; - cleanup_repair_output(&fixed_path).await; + segment_cache.remove_staged(output).await?; self.record_metadata(identity.clone(), status, raw_size, raw_size, Some(reason)).await; return Ok(None); } @@ -309,7 +342,7 @@ impl HlsSegmentRepairManager { Ok(scan) => scan, Err(reason) => { debug_repair_event(context, executed_level, "validation probe failed", Some(&reason)); - cleanup_repair_output(&fixed_path).await; + segment_cache.remove_staged(output).await?; self.record_metadata( identity.clone(), RepairStatus::ValidationFailed, @@ -325,7 +358,7 @@ impl HlsSegmentRepairManager { validate_repair(&runtime.config, codec, &raw_scan, &fixed_scan, executed_level, &stream_selection); if let Err(reason) = validation { debug_repair_event(context, executed_level, "validation failed", Some(&reason)); - cleanup_repair_output(&fixed_path).await; + segment_cache.remove_staged(output).await?; self.record_metadata(identity.clone(), RepairStatus::ValidationFailed, raw_size, raw_size, Some(reason)) .await; return Ok(None); @@ -339,7 +372,8 @@ impl HlsSegmentRepairManager { executed_level.as_log_value(), started.elapsed().as_millis() ); - Ok(Some(fixed_path)) + output.size = fs::metadata(&fixed_path).await?.len(); + Ok(Some(output)) } } @@ -390,6 +424,8 @@ async fn run_remux( mode: HlsSegmentRepairMode, stream_selection: &RepairRemuxStreamSelection, deadline: &HlsPostProcessingDeadline, + disk_budget: super::RepairDiskBudget<'_>, + staged: &StagedCacheObject, ) -> Result<(), String> { let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; @@ -417,7 +453,7 @@ async fn run_remux( .map(ToOwned::to_owned), ); let args = args.iter().map(String::as_str).collect::>(); - run_command_with_deadline("ffmpeg", &args, deadline).await.map(|_| ()) + super::run_reserved_remux(&args[..args.len() - 1], staged, disk_budget, deadline).await } pub(crate) async fn run_command_with_deadline( diff --git a/backend/hls/src/segment_repair/tests/transport.rs b/backend/hls/src/segment_repair/tests/transport.rs index 199636976..2a723d0d6 100644 --- a/backend/hls/src/segment_repair/tests/transport.rs +++ b/backend/hls/src/segment_repair/tests/transport.rs @@ -16,6 +16,7 @@ fn repair_prewarm_candidates_include_only_ready_clear_origin_ts_media() { host_local_index: 0, }, proxy_seq, + media_start_ms: 0, duration_ms: 6_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/hls/src/segment_revision.rs b/backend/hls/src/segment_revision.rs index 85c6d49e7..7a624b599 100644 --- a/backend/hls/src/segment_revision.rs +++ b/backend/hls/src/segment_revision.rs @@ -279,12 +279,14 @@ impl SegmentRevisionStore { Ok(()) } - pub fn retire_unpinned(&self) -> io::Result>> { + pub fn retire_unpinned(&self) -> io::Result>> { self.retire_unpinned_limit(128) } + + pub(crate) fn retire_unpinned_limit(&self, limit: usize) -> io::Result>> { let mut entries = self.entries.lock().map_err(|_| io::Error::other("revision registry poisoned"))?; let ids: Vec<_> = entries .iter() .filter_map(|(id, revision)| (revision.pins.load(Ordering::Acquire) == 0).then_some(*id)) - .take(128) + .take(limit) .collect(); Ok(ids.into_iter().filter_map(|id| entries.remove(&id)).collect()) } diff --git a/backend/hls/src/segment_watchdog.rs b/backend/hls/src/segment_watchdog.rs index f589532d6..ce3d0380c 100644 --- a/backend/hls/src/segment_watchdog.rs +++ b/backend/hls/src/segment_watchdog.rs @@ -275,8 +275,16 @@ impl HlsCorruptSegmentWatchdogManager { return segment_cache.commit_staged(key, raw).await; } let fixed_path = watchdog_output_path(&raw.path); - if let Err(reason) = sanitize_corrupt_segment(&raw.path, &fixed_path, deadline).await { - cleanup_watchdog_output(&fixed_path).await; + let mut fixed = + segment_cache.adopt_staged_file(fixed_path.clone(), 0)?.with_write_budget(raw.write_budget.clone())?; + let disk_budget = raw.write_budget.as_ref().map(|reservation| crate::segment_repair::RepairDiskBudget { + cache: segment_cache, + reservation, + session: &context.proxy_session_id, + retained_staging_bytes: raw.size, + }); + if let Err(reason) = sanitize_corrupt_segment(&raw.path, &fixed_path, deadline, &fixed, disk_budget).await { + segment_cache.remove_staged(fixed).await?; let status = if reason == "timeout" { HlsCorruptSegmentWatchdogStatus::Timeout } else { @@ -289,7 +297,7 @@ impl HlsCorruptSegmentWatchdogManager { Ok(metadata) => metadata.len(), Err(err) => { let reason = format!("metadata_failed:{err}"); - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, @@ -304,7 +312,7 @@ impl HlsCorruptSegmentWatchdogManager { let fixed_detect = match detect_packet_corrupt(&fixed_path, deadline).await { Ok(count) => count, Err(reason) if reason == "timeout" => { - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::Timeout, @@ -316,7 +324,7 @@ impl HlsCorruptSegmentWatchdogManager { return segment_cache.commit_staged(key, raw).await; } Err(reason) => { - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, @@ -339,7 +347,7 @@ impl HlsCorruptSegmentWatchdogManager { ) .await; if let Err(reason) = validation { - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, @@ -351,7 +359,8 @@ impl HlsCorruptSegmentWatchdogManager { return segment_cache.commit_staged(key, raw).await; } let raw_size = raw.size; - let committed = match commit_watchdog_output_or_raw(segment_cache, key, raw, fixed_path, fixed_size).await? { + fixed.size = fixed_size; + let committed = match commit_watchdog_staged_or_raw(segment_cache, key, raw, Ok(fixed)).await? { HlsWatchdogCommitOutcome::Sanitized(committed) => committed, HlsWatchdogCommitOutcome::RawFallback(committed) => return Ok(committed), }; @@ -485,40 +494,40 @@ async fn sanitize_corrupt_segment( input_path: &Path, output_path: &Path, deadline: &HlsPostProcessingDeadline, + staged: &StagedCacheObject, + disk_budget: Option>, ) -> Result<(), String> { let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; - run_command_with_deadline( - "ffmpeg", - &[ - "-hide_banner", - "-nostdin", - "-y", - "-fflags", - "+discardcorrupt", - "-copyts", - "-i", - input, - "-map", - "0", - "-c", - "copy", - "-mpegts_flags", - "+resend_headers", - "-mpegts_copyts", - "1", - "-muxpreload", - "0", - "-muxdelay", - "0", - "-f", - "mpegts", - output, - ], - deadline, - ) - .await - .map(|_| ()) + let args = [ + "-hide_banner", + "-nostdin", + "-y", + "-fflags", + "+discardcorrupt", + "-copyts", + "-i", + input, + "-map", + "0", + "-c", + "copy", + "-mpegts_flags", + "+resend_headers", + "-mpegts_copyts", + "1", + "-muxpreload", + "0", + "-muxdelay", + "0", + "-f", + "mpegts", + output, + ]; + if let Some(budget) = disk_budget { + return crate::segment_repair::run_reserved_remux(&args[..args.len() - 1], staged, budget, deadline).await; + } + run_command_with_deadline("ffmpeg", &args, deadline).await.map(|_| ()) } async fn validate_sanitized_segment( @@ -661,6 +670,7 @@ fn watchdog_output_path(input_path: &Path) -> PathBuf { PathBuf::from(value) } +#[cfg(test)] async fn cleanup_watchdog_output(path: &Path) { match fs::remove_file(path).await { Ok(()) => {} @@ -669,6 +679,7 @@ async fn cleanup_watchdog_output(path: &Path) { } } +#[cfg(test)] async fn adopt_watchdog_output( segment_cache: &HlsSegmentCache, path: PathBuf, @@ -683,6 +694,7 @@ async fn adopt_watchdog_output( } } +#[cfg(test)] async fn commit_watchdog_output_or_raw( segment_cache: &HlsSegmentCache, key: &K, @@ -693,23 +705,33 @@ async fn commit_watchdog_output_or_raw( where K: HlsCacheObjectKey, { - let fixed = match adopt_watchdog_output(segment_cache, fixed_path, fixed_size).await { + let fixed = adopt_watchdog_output(segment_cache, fixed_path, fixed_size).await; + commit_watchdog_staged_or_raw(segment_cache, key, raw, fixed).await +} + +async fn commit_watchdog_staged_or_raw( + segment_cache: &HlsSegmentCache, + key: &K, + raw: StagedCacheObject, + fixed: io::Result, +) -> io::Result { + let fixed = match fixed.and_then(|fixed| fixed.with_write_budget(raw.write_budget.clone())) { Ok(fixed) => fixed, Err(error) => { warn!("HLS watchdog fixed staging adoption failed; committing raw fallback: error_kind={:?}", error.kind()); return segment_cache.commit_staged(key, raw).await.map(HlsWatchdogCommitOutcome::RawFallback); } }; + let fixed_cleanup = fixed.clone(); let committed = match segment_cache.commit_staged(key, fixed).await { Ok(committed) => committed, Err(error) => { warn!("HLS watchdog fixed staging commit failed; committing raw fallback: error_kind={:?}", error.kind()); + segment_cache.remove_staged(fixed_cleanup).await?; return segment_cache.commit_staged(key, raw).await.map(HlsWatchdogCommitOutcome::RawFallback); } }; - if let Err(error) = segment_cache.remove_staged(raw).await { - warn!("HLS watchdog raw staging cleanup deferred: error_kind={:?}", error.kind()); - } + segment_cache.remove_staged(raw).await?; Ok(HlsWatchdogCommitOutcome::Sanitized(committed)) } @@ -763,8 +785,8 @@ fn debug_watchdog_event( #[cfg(test)] mod tests { use super::{ - commit_watchdog_output_or_raw, count_packet_corrupt_events, parse_packet_corrupt_dts, - unsupported_sanitize_extension, HlsWatchdogCommitOutcome, + commit_watchdog_output_or_raw, commit_watchdog_staged_or_raw, count_packet_corrupt_events, + parse_packet_corrupt_dts, unsupported_sanitize_extension, HlsWatchdogCommitOutcome, }; use crate::{HlsSegmentCache, ProxySessionId, SegmentCacheKey}; use std::time::Duration; @@ -849,4 +871,33 @@ Last message repeated 3 times assert_eq!(tokio::fs::read(committed.path).await.expect("committed fallback reads"), b"raw"); assert!(!fixed_path.exists()); } + + #[tokio::test] + async fn failed_raw_staging_cleanup_aborts_postprocessing_before_further_publication() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(40, 40); + let key = SegmentCacheKey::new(ProxySessionId("cleanup-failure".into()), 1, "ts"); + let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + let raw = cache.stage_temp_with_budget(&key, &[1; 10][..], deadline, Some(budget.clone())).await?; + let fixed = cache.stage_temp_with_budget(&key, &[2; 20][..], deadline, Some(budget.clone())).await?; + cache.fail_next_cache_deletes(raw.path.clone(), 1); + + let error = commit_watchdog_staged_or_raw(&cache, &key, raw.clone(), Ok(fixed)) + .await + .err() + .expect("failed temp removal must stop the owning startup transaction"); + + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + assert_eq!(tokio::fs::read(&raw.path).await?, [1; 10]); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 20); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10); + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [2; 20]); + cache.remove_staged(raw).await?; + drop(budget); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert!(!cache.has_active_temp_files()); + Ok(()) + } } diff --git a/backend/hls/src/session.rs b/backend/hls/src/session.rs index 14beb4aa3..5e0211392 100644 --- a/backend/hls/src/session.rs +++ b/backend/hls/src/session.rs @@ -120,6 +120,7 @@ pub struct HlsSessionOriginControl { host_local_highwater_observation_order: u64, pub acceptance_episode: Option, pub recovery_samples: HlsBoundedRecoverySamples, + pub(crate) qualification: crate::short_manifest::HlsOriginQualification, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -146,6 +147,7 @@ impl Default for HlsSessionOriginControl { host_local_highwater_observation_order: 0, acceptance_episode: None, recovery_samples: HlsBoundedRecoverySamples::default(), + qualification: crate::short_manifest::HlsOriginQualification::default(), } } } @@ -155,6 +157,18 @@ impl HlsSessionOriginControl { pub fn record_origin_response(&mut self, now_ms: u64) { self.last_origin_response_at_ms = Some(now_ms); } + /// Includes publication lateness even when the most recent HTTP request has not failed yet. + pub(crate) fn path_is_degraded_at(&self, now_ms: u64) -> bool { + self.path_condition.is_degraded() + || self.last_media_progress_at_ms.zip(self.target_duration_snapshot_ms).is_some_and( + |(last_progress_ms, target_duration_ms)| { + target_duration_ms > 0 + && now_ms.saturating_sub(last_progress_ms) + >= crate::origin_progress::publication_late_after_ms(target_duration_ms) + }, + ) + } + pub fn record_media_progress(&mut self, now_ms: u64, target_duration_ms: u64) { self.last_origin_response_at_ms = Some(now_ms); self.last_media_progress_at_ms = Some(now_ms); @@ -253,6 +267,9 @@ pub struct HlsSession { pub origin_account_binding: Option, pub origin_account_io_lease: Option>, pub origin_account_rebind: HlsOriginAccountRebindState, + pub origin_account_prepare_lock: Arc>, + pub origin_account_runtime: Option>, + pub origin_account_manifest_pending: Option, pub effective_origin_acquire_policy: Option, pub mode: HlsSessionMode, pub transient: TransientPassthroughState, @@ -286,6 +303,8 @@ pub struct HlsSession { pub origin_epoch_sequence_base: Option, pub origin_seq_highwater: Option, pub proxy_next_seq: Option, + /// End of all appended media, retained even when every old cache entry is collected. + pub next_media_start_ms: u64, pub origin_to_proxy: HashMap, pub published_resource_history: HlsPublishedResourceHistory, pub discontinuity_sequence: u64, @@ -330,6 +349,9 @@ impl HlsSession { origin_account_binding: None, origin_account_io_lease: None, origin_account_rebind: HlsOriginAccountRebindState::default(), + origin_account_prepare_lock: Arc::new(tokio::sync::Mutex::new(())), + origin_account_runtime: None, + origin_account_manifest_pending: None, effective_origin_acquire_policy: None, mode: HlsSessionMode::NormalCacheTimeline, transient: TransientPassthroughState::default(), @@ -360,6 +382,7 @@ impl HlsSession { origin_epoch_sequence_base: None, origin_seq_highwater: None, proxy_next_seq: None, + next_media_start_ms: 0, origin_to_proxy: HashMap::new(), published_resource_history: HlsPublishedResourceHistory::default(), discontinuity_sequence: 0, @@ -466,6 +489,43 @@ impl HlsSession { pub fn is_gc_marked_for_removal(&self) -> bool { self.gc_marked_for_removal } + /// Invalidates account-dependent work while preserving published media and host/content anchors. + pub fn handoff_origin_account(&mut self, binding: HlsOriginAccountBinding) { + self.invalidate_queued_origin_work(); + self.origin_account_io_lease = None; + self.origin_provider_session_headers.clear(); + self.origin_control.manifest_origin_binding = None; + self.origin_control.acceptance_episode = None; + self.origin_control.acceptance_generation.0 = self.origin_control.acceptance_generation.0.saturating_add(1); + self.origin_control.qualification = crate::short_manifest::HlsOriginQualification::default(); + self.origin_refresh = OriginRefreshState::default(); + self.origin_account_manifest_pending = Some(binding.generation); + self.transient.invalidate_origin_resources(); + for segment in self.segments.values_mut() { + if !matches!(segment.status, SegmentCacheStatus::Ready { .. }) { + segment.origin_fetch_ref = None; + if !matches!(segment.status, SegmentCacheStatus::Fetching { .. }) { + segment.status = SegmentCacheStatus::Discovered; + } + } + } + for map in self.maps.values_mut() { + if !matches!(map.status, MapCacheStatus::Ready { .. }) { + map.origin_fetch_ref = None; + if !matches!(map.status, MapCacheStatus::Fetching { .. }) { + map.status = MapCacheStatus::Discovered; + } + } + } + self.replace_origin_account_binding(Some(binding)); + } + + pub(crate) fn origin_handoff_ready_after(&self, generation: u64) -> bool { + self.activity.origin_work_generation != generation + && self.origin_account_manifest_pending.is_none() + && !self.is_gc_marked_for_removal() + } + pub fn replace_origin_account_binding(&mut self, binding: Option) { let binding_changed = match (&self.origin_account_binding, &binding) { (Some(current), Some(next)) => { @@ -778,6 +838,15 @@ impl HlsSession { }; } + /// Provider lease kind retained across account replacement and subsequent origin I/O. + pub const fn origin_playback_kind(&self) -> tuliprox_core::model::PlaybackKind { + if self.key.archive_reference.is_some() { + tuliprox_core::model::PlaybackKind::Catchup + } else { + tuliprox_core::model::PlaybackKind::LiveHls + } + } + pub fn start_origin_work(&mut self) -> u64 { self.activity.active_origin_work_count.fetch_add(1, std::sync::atomic::Ordering::AcqRel); self.activity.origin_work_generation @@ -828,6 +897,9 @@ impl fmt::Debug for HlsSession { .field("origin_account_binding", &self.origin_account_binding) .field("origin_account_io_lease", &self.origin_account_io_lease) .field("origin_account_rebind", &self.origin_account_rebind) + .field("origin_account_prepare_lock", &self.origin_account_prepare_lock) + .field("origin_account_runtime", &self.origin_account_runtime) + .field("origin_account_manifest_pending", &self.origin_account_manifest_pending) .field("effective_origin_acquire_policy", &self.effective_origin_acquire_policy) .field("mode", &self.mode) .field("transient", &self.transient) @@ -858,6 +930,7 @@ impl fmt::Debug for HlsSession { .field("origin_epoch_sequence_base", &self.origin_epoch_sequence_base) .field("origin_seq_highwater", &self.origin_seq_highwater) .field("proxy_next_seq", &self.proxy_next_seq) + .field("next_media_start_ms", &self.next_media_start_ms) .field("origin_to_proxy_len", &self.origin_to_proxy.len()) .field("published_resource_history", &self.published_resource_history.generation()) .field("discontinuity_sequence", &self.discontinuity_sequence) @@ -934,6 +1007,7 @@ mod tests { host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX), }, proxy_seq, + media_start_ms: proxy_seq * duration_ms, duration_ms, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -1713,4 +1787,44 @@ mod tests { assert_eq!(fallback.connection_kind, ConnectionKind::Normal); assert_eq!(fallback.priority, 0); } + #[test] + fn account_handoff_invalidates_work_and_recovery_but_preserves_timeline_and_archive() { + let mut session = recovery_eligible_session(); + session.key = session.key.with_archive_reference(123).with_archive_identity("archive-identity"); + session.origin_source = crate::HlsOriginSource::from_session_key(&session.key); + let source = session.origin_source.clone(); + let entry = ready_segment(&session, 10, 0, 4_000, 32); + session.segments.insert(10, entry); + session.activity.active_access_lease_count = 2; + let key = session.key.clone(); + let proxy_id = session.proxy_session_id.clone(); + let rendered = session.last_rendered_manifest.clone(); + let pinned_host = session.origin_control.pinned_host.clone(); + session + .origin_provider_session_headers + .insert(axum::http::header::COOKIE, axum::http::HeaderValue::from_static("old=secret")); + let old_work = session.start_origin_work(); + let binding = crate::origin::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("replacement"), + "owner".to_string(), + 3, + 10, + ); + session.handoff_origin_account(binding); + assert!(!session.finish_origin_work(old_work)); + assert_eq!(session.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), 0); + assert_eq!((&session.key, &session.proxy_session_id), (&key, &proxy_id)); + assert_eq!(session.last_rendered_manifest, rendered); + assert_eq!(session.origin_control.pinned_host, pinned_host); + assert_eq!(session.origin_seq_highwater, Some(10)); + assert_eq!(session.activity.active_access_lease_count, 2); + assert_eq!(session.origin_source, source); + assert_eq!(session.origin_playback_kind(), tuliprox_core::model::PlaybackKind::Catchup); + assert_eq!(session.origin_account_manifest_pending, Some(3)); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert!(session.origin_control.acceptance_episode.is_none()); + assert!(session.origin_provider_session_headers.is_empty()); + assert!(matches!(session.segments[&10].status, SegmentCacheStatus::Ready { content_length: 32, .. })); + } } diff --git a/backend/hls/src/short_manifest.rs b/backend/hls/src/short_manifest.rs new file mode 100644 index 000000000..0888f1eec --- /dev/null +++ b/backend/hls/src/short_manifest.rs @@ -0,0 +1,214 @@ +//! Session-owned permission for an origin's unusually short live playlists. + +use crate::{ + manifest_acceptance::HlsManifestAcceptanceGeneration, manifest_origin_binding::HlsManifestOriginBinding, + media_reserve::HlsManifestCommitIdentity, HlsSession, +}; + +/// The validated origin foundation, deliberately independent of window movement and leases. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct HlsOriginQualificationBasis { + epoch: u64, + work_generation: u64, + binding: Option, + host: Option, +} + +impl HlsOriginQualificationBasis { + pub(crate) fn capture(session: &HlsSession) -> Self { + Self { + epoch: session.origin_epoch, + work_generation: session.activity.origin_work_generation, + binding: session.origin_control.manifest_origin_binding.clone(), + host: session.origin_control.pinned_host.clone(), + } + } +} + +/// Window sizes accepted for the current validated origin foundation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum HlsQualifiedOriginWindows { + Regular, + ShortAllowed, +} + +/// One shared qualification and its retry receipt; never consumed by a publication. +#[derive(Clone, Debug, Default)] +pub(crate) enum HlsOriginQualification { + #[default] + Unchecked, + Checking { + basis: HlsOriginQualificationBasis, + generation: HlsManifestAcceptanceGeneration, + target: HlsManifestOriginBinding, + }, + Qualified { + basis: HlsOriginQualificationBasis, + generation: HlsManifestAcceptanceGeneration, + windows: HlsQualifiedOriginWindows, + /// Last publication when this origin foundation was qualified. + publication_boundary: Option, + }, + RetryPending { + basis: HlsOriginQualificationBasis, + generation: HlsManifestAcceptanceGeneration, + target: HlsManifestOriginBinding, + }, +} + +impl HlsOriginQualification { + pub(crate) fn qualified_windows(&self, basis: &HlsOriginQualificationBasis) -> Option { + match self { + Self::Qualified { basis: current, windows, .. } if current == basis => Some(*windows), + Self::Unchecked | Self::Checking { .. } | Self::Qualified { .. } | Self::RetryPending { .. } => None, + } + } + + pub(crate) fn pending_is_current(&self, basis: &HlsOriginQualificationBasis) -> bool { + match self { + Self::Checking { basis: current, .. } | Self::RetryPending { basis: current, .. } => current == basis, + Self::Unchecked | Self::Qualified { .. } => true, + } + } + + pub(crate) fn retry_target(&self, basis: &HlsOriginQualificationBasis) -> Option<&HlsManifestOriginBinding> { + match self { + Self::RetryPending { basis: current, target, .. } if current == basis => Some(target), + Self::Unchecked | Self::Checking { .. } | Self::Qualified { .. } | Self::RetryPending { .. } => None, + } + } + + pub(crate) fn record_failure(&mut self) { + if let Self::Checking { basis, generation, target } = self { + *self = Self::RetryPending { basis: basis.clone(), generation: *generation, target: target.clone() }; + } + } + + pub(crate) const fn generation(&self) -> Option { + match self { + Self::Unchecked => None, + Self::Checking { generation, .. } + | Self::Qualified { generation, .. } + | Self::RetryPending { generation, .. } => Some(*generation), + } + } +} + +impl HlsSession { + pub(crate) fn origin_qualification_retry_target(&self) -> Option<&HlsManifestOriginBinding> { + // Deterministic conflicts retain their own ordinary-sample path. Only + // changed receipt evidence may authorize a replacement full burst. + if self + .origin_control + .acceptance_episode + .as_ref() + .is_some_and(|episode| episode.deterministic_conflict_receipt().is_some()) + { + return None; + } + self.origin_control.qualification.retry_target(&HlsOriginQualificationBasis::capture(self)) + } + + pub(crate) fn origin_is_qualified(&self) -> bool { + self.origin_control.qualification.qualified_windows(&HlsOriginQualificationBasis::capture(self)).is_some() + } + + pub(crate) fn origin_qualification_awaits_publication(&self) -> bool { + match &self.origin_control.qualification { + HlsOriginQualification::Qualified { basis, publication_boundary, .. } + if basis == &HlsOriginQualificationBasis::capture(self) => + { + self.last_normal_manifest_commit_identity() == *publication_boundary + } + HlsOriginQualification::Unchecked + | HlsOriginQualification::Checking { .. } + | HlsOriginQualification::Qualified { .. } + | HlsOriginQualification::RetryPending { .. } => false, + } + } + + pub(crate) fn allows_short_origin_windows(&self) -> bool { + self.origin_control.qualification.qualified_windows(&HlsOriginQualificationBasis::capture(self)) + == Some(HlsQualifiedOriginWindows::ShortAllowed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::HlsSessionKey; + + #[test] + fn permission_survives_progress_but_not_origin_foundation_changes() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "short"), b"secret", 0); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: HlsOriginQualificationBasis::capture(&session), + generation: HlsManifestAcceptanceGeneration(1), + windows: HlsQualifiedOriginWindows::ShortAllowed, + publication_boundary: None, + }; + for progress in 1..=10 { + session.origin_seq_highwater = Some(progress); + session.origin_control.record_media_progress(progress, 4_000); + assert!(session.allows_short_origin_windows()); + } + session.origin_epoch += 1; + assert!(!session.allows_short_origin_windows()); + session.origin_epoch -= 1; + session.activity.origin_work_generation += 1; + assert!(!session.allows_short_origin_windows()); + session.activity.origin_work_generation -= 1; + session.origin_control.pinned_host = Some("changed.example".to_string()); + assert!(!session.allows_short_origin_windows()); + } + + #[test] + fn unchecked_regular_checking_and_failed_results_do_not_permit_short_publication() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "short"), b"secret", 0); + assert!(!session.allows_short_origin_windows()); + let basis = HlsOriginQualificationBasis::capture(&session); + let generation = HlsManifestAcceptanceGeneration(1); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: basis.clone(), + generation, + windows: HlsQualifiedOriginWindows::Regular, + publication_boundary: None, + }; + assert!(!session.allows_short_origin_windows()); + session.origin_control.qualification = HlsOriginQualification::Checking { + basis, + generation, + target: HlsManifestOriginBinding::new(url::Url::parse("https://origin.example/live.m3u8").unwrap(), None) + .unwrap(), + }; + assert!(!session.allows_short_origin_windows()); + session.origin_control.qualification.record_failure(); + assert!(!session.allows_short_origin_windows()); + } + + #[test] + fn pending_publication_tracks_current_qualification_and_commit_generation() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "revalidation"), b"secret", 0); + let first = session.next_manifest_commit_identity(10).unwrap(); + session.record_normal_manifest_commit_identity(first); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: HlsOriginQualificationBasis::capture(&session), + generation: HlsManifestAcceptanceGeneration(2), + windows: HlsQualifiedOriginWindows::ShortAllowed, + publication_boundary: Some(first), + }; + assert!(session.origin_qualification_awaits_publication()); + let next = session.next_manifest_commit_identity(10).unwrap(); + session.record_normal_manifest_commit_identity(next); + assert!(!session.origin_qualification_awaits_publication(), "equal timestamps still identify distinct commits"); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: HlsOriginQualificationBasis::capture(&session), + generation: HlsManifestAcceptanceGeneration(3), + windows: HlsQualifiedOriginWindows::ShortAllowed, + publication_boundary: Some(next), + }; + assert!(session.origin_qualification_awaits_publication()); + session.origin_epoch += 1; + assert!(!session.origin_qualification_awaits_publication()); + } +} diff --git a/backend/hls/src/terminal_commit.rs b/backend/hls/src/terminal_commit.rs index b11f693a7..6b07edb18 100644 --- a/backend/hls/src/terminal_commit.rs +++ b/backend/hls/src/terminal_commit.rs @@ -154,6 +154,7 @@ pub(super) type HlsTerminalCommitAttemptFn = #[derive(Debug, Clone)] pub enum HlsTerminalLeaseDecision { Tail(Arc), + CustomResponse(Arc), Unavailable(HlsTerminalTailCompatibility), UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility), } @@ -303,6 +304,16 @@ impl HlsTerminalCommitCommand { self.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() } + fn submission_deadline_ms(&self, now_ms: u64) -> u64 { + if self.has_independent_commit_deadline() { + // No live media is spliced by this decision. It gets one bounded CAS budget, + // even when the media deadline has already elapsed; repeated submissions cannot renew it. + now_ms.saturating_add(terminal_commit_retry_schedule_budget_ms()) + } else { + self.latest_safe_terminal_commit_at_ms() + } + } + /// Monotonic session/lease evidence used to prevent a late stale request /// from replacing a newer command for the same lease decision generation. fn evidence_version(&self) -> [u64; 7] { @@ -317,8 +328,11 @@ impl HlsTerminalCommitCommand { ] } - fn is_unavailable_after_owner_failure(&self) -> bool { - matches!(&self.decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) + fn has_independent_commit_deadline(&self) -> bool { + matches!( + &self.decision, + HlsTerminalLeaseDecision::CustomResponse(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) + ) } fn may_replace(&self, current: &Self) -> bool { @@ -329,8 +343,8 @@ impl HlsTerminalCommitCommand { return self.session_incarnation > current.session_incarnation; } let evidence_order = self.evidence_version().cmp(¤t.evidence_version()); - if self.is_unavailable_after_owner_failure() - && !current.is_unavailable_after_owner_failure() + if self.has_independent_commit_deadline() + && !current.has_independent_commit_deadline() && evidence_order != std::cmp::Ordering::Less { return true; @@ -359,6 +373,7 @@ impl std::fmt::Debug for HlsTerminalCommitCommand { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let decision_kind = match &self.decision { HlsTerminalLeaseDecision::Tail(_) => "tail", + HlsTerminalLeaseDecision::CustomResponse(_) => "custom-response", HlsTerminalLeaseDecision::Unavailable(_) => "unavailable", HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) => "unavailable-after-owner-failure", }; @@ -656,7 +671,7 @@ impl HlsTerminalCommitRetryCoordinator { return HlsTerminalCommitSubmissionDecision::Cancelled; } let key = command.key.clone(); - let incoming_deadline_ms = command.latest_safe_terminal_commit_at_ms(); + let incoming_deadline_ms = command.submission_deadline_ms(now_ms); if let Some(existing) = state.owners.get(&key).map(HlsTerminalCommitExistingOwner::snapshot) { return self.submit_existing(&mut state, command, &key, now_ms, incoming_deadline_ms, existing); } @@ -672,8 +687,21 @@ impl HlsTerminalCommitRetryCoordinator { incoming_deadline_ms: u64, existing: HlsTerminalCommitExistingOwner, ) -> HlsTerminalCommitSubmissionDecision { - let effective_deadline_ms = existing.effective_deadline_ms.min(incoming_deadline_ms); - if !command.may_replace(&existing.command) { + let may_replace = command.may_replace(&existing.command); + let changes_deadline_kind = + command.has_independent_commit_deadline() != existing.command.has_independent_commit_deadline(); + let replaces_splice_with_independent_commit = + changes_deadline_kind && may_replace && command.has_independent_commit_deadline(); + let effective_deadline_ms = if changes_deadline_kind { + if may_replace { + incoming_deadline_ms + } else { + existing.effective_deadline_ms + } + } else { + existing.effective_deadline_ms.min(incoming_deadline_ms) + }; + if !may_replace { let deadline_tightened = effective_deadline_ms < existing.effective_deadline_ms; let tightened_scheduled_at_ms = if deadline_tightened { terminal_commit_scheduled_before_deadline(existing.scheduled_at_ms, now_ms, effective_deadline_ms) @@ -732,7 +760,8 @@ impl HlsTerminalCommitRetryCoordinator { if let Some(owner) = state.owners.get_mut(key) { owner.owner_token = owner_token; owner.command = Arc::clone(&command); - owner.attempts_completed = existing.attempts_completed.max(1); + owner.attempts_completed = + if replaces_splice_with_independent_commit { 1 } else { existing.attempts_completed.max(1) }; owner.effective_deadline_ms = effective_deadline_ms; owner.scheduled_at_ms = now_ms; owner.scheduling_sequence = scheduling_sequence; @@ -1582,7 +1611,7 @@ mod tests { } #[test] - fn unavailable_after_owner_failure_retains_the_exclusive_media_tail_deadline() { + fn unavailable_after_owner_failure_has_a_separate_bounded_cas_deadline() { let mut command = retry_command("failed-closed-unavailable", 10_000, 1, 1); let media_tail_deadline = command.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); @@ -1591,10 +1620,11 @@ mod tests { command.decision = HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility::TerminalMediaNotReady); assert_eq!(command.latest_safe_terminal_commit_at_ms(), media_tail_deadline); + assert_eq!(command.submission_deadline_ms(20_000), 20_000 + super::terminal_commit_retry_schedule_budget_ms()); } #[test] - fn failed_media_tail_owner_cannot_be_replaced_at_the_exclusive_deadline() { + fn failed_media_tail_owner_can_finish_unavailable_at_the_exclusive_deadline() { let coordinator = HlsTerminalCommitRetryCoordinator::default(); let command = retry_command("failed-owner-replacement", 10_000, 1, 1); let key = command.key.clone(); @@ -1608,7 +1638,7 @@ mod tests { coordinator.fail_owner(&key, owner_token, HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed); let replacement = HlsTerminalCommitCommand { - key, + key: key.clone(), session, session_incarnation, preparation, @@ -1622,8 +1652,11 @@ mod tests { }; assert!(matches!( coordinator.submit(replacement, failed_at_ms), - HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + HlsTerminalCommitSubmissionDecision::Attempt { .. } )); + let owner = coordinator.owner_snapshot(&key).expect("unavailable CAS owner"); + assert_eq!(owner.effective_deadline_ms, failed_at_ms + super::terminal_commit_retry_schedule_budget_ms()); + assert_eq!(owner.attempts_completed, 1); } #[test] @@ -1897,4 +1930,35 @@ mod tests { assert!(!coordinator.owner_snapshot(&first_key).expect("released worker owner").in_flight); assert!(coordinator.owner_snapshot(&second_key).expect("unrelated client owner").in_flight); } + + #[test] + fn unavailable_cas_budget_is_neither_renewed_by_polls_nor_poisoned_by_stale_media_deadlines() { + let coordinator = HlsTerminalCommitRetryCoordinator::default(); + let mut initial = retry_command("bounded-unavailable", 10_000, 1, 1); + initial.decision = + HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility::TerminalMediaNotReady); + let session = Arc::clone(&initial.session); + let key = initial.key.clone(); + assert!(matches!(coordinator.submit(initial, 10_000), HlsTerminalCommitSubmissionDecision::Attempt { .. })); + let deadline = coordinator.owner_snapshot(&key).expect("initial owner").effective_deadline_ms; + let mut stale_media = retry_command("bounded-unavailable", 10_000, 2, 1); + stale_media.session = Arc::clone(&session); + assert!(matches!( + coordinator.submit(stale_media, 10_100), + HlsTerminalCommitSubmissionDecision::PendingExisting { .. } + )); + assert_eq!( + coordinator.owner_snapshot(&key).expect("unavailable owner survives stale media").effective_deadline_ms, + deadline + ); + let mut duplicate = retry_command("bounded-unavailable", 10_000, 3, 1); + duplicate.session = session; + duplicate.decision = + HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility::TerminalMediaNotReady); + assert!(matches!( + coordinator.submit(duplicate, deadline), + HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + )); + assert_eq!(coordinator.owner_snapshot(&key).expect("bounded owner").effective_deadline_ms, deadline); + } } diff --git a/backend/hls/src/terminal_tail/plan.rs b/backend/hls/src/terminal_tail/plan.rs index 33ff5bd77..c92863d28 100644 --- a/backend/hls/src/terminal_tail/plan.rs +++ b/backend/hls/src/terminal_tail/plan.rs @@ -195,6 +195,7 @@ pub enum HlsLeasePlaybackMode { #[default] Live, TerminalTail(Arc), + TerminalCustomResponse(Arc), TerminalUnavailable { decision_generation: u64, reason: HlsTerminalTailCompatibility, diff --git a/backend/hls/src/terminal_tail/tests.rs b/backend/hls/src/terminal_tail/tests.rs index 2eafb74ae..5c07facb0 100644 --- a/backend/hls/src/terminal_tail/tests.rs +++ b/backend/hls/src/terminal_tail/tests.rs @@ -162,6 +162,7 @@ async fn unresolved_aes_terminal_base_evidence(iv: &str) -> HlsTerminalBaseEvide host_local_index: 0, }, proxy_seq, + media_start_ms: 0, duration_ms: 4_000, proxy_file_ext: "ts".into(), content_type: "video/mp2t".into(), @@ -238,6 +239,7 @@ async fn terminal_timing_session( host_local_index: 0, }, proxy_seq, + media_start_ms: proxy_seq * terminal_asset.duration_ms(), duration_ms: terminal_asset.duration_ms(), proxy_file_ext: "ts".into(), content_type: "video/mp2t".into(), diff --git a/backend/hls/src/timeline.rs b/backend/hls/src/timeline.rs index 08150830f..1d3072259 100644 --- a/backend/hls/src/timeline.rs +++ b/backend/hls/src/timeline.rs @@ -172,6 +172,8 @@ pub fn is_hls_provisioning_gap_segment(entry: &SegmentEntry) -> bool { pub struct SegmentEntry { pub origin_key: OriginSegmentKey, pub proxy_seq: u64, + /// Immutable session-local media offset; independent of the retained cache window. + pub media_start_ms: u64, pub duration_ms: u64, pub proxy_file_ext: String, pub content_type: String, @@ -193,6 +195,7 @@ impl fmt::Debug for SegmentEntry { f.debug_struct("SegmentEntry") .field("origin_key", &self.origin_key) .field("proxy_seq", &self.proxy_seq) + .field("media_start_ms", &self.media_start_ms) .field("duration_ms", &self.duration_ms) .field("proxy_file_ext", &self.proxy_file_ext) .field("content_type", &self.content_type) @@ -317,6 +320,7 @@ struct TimelineDraft { origin_epoch_sequence_base: Option, origin_seq_highwater: Option, proxy_next_seq: Option, + next_media_start_ms: u64, origin_to_proxy: HashMap, published_resource_history: HlsPublishedResourceHistory, discontinuity_sequence: u64, @@ -364,6 +368,7 @@ impl From<&HlsSession> for TimelineDraft { origin_epoch_sequence_base: session.origin_epoch_sequence_base, origin_seq_highwater: session.origin_seq_highwater, proxy_next_seq: session.proxy_next_seq, + next_media_start_ms: session.next_media_start_ms, origin_to_proxy: session.origin_to_proxy.clone(), published_resource_history: session.published_resource_history.clone(), discontinuity_sequence: session.discontinuity_sequence, @@ -482,6 +487,7 @@ impl HlsSession { self.origin_epoch_sequence_base = draft.origin_epoch_sequence_base; self.origin_seq_highwater = draft.origin_seq_highwater; self.proxy_next_seq = draft.proxy_next_seq; + self.next_media_start_ms = draft.next_media_start_ms; self.origin_to_proxy = draft.origin_to_proxy; self.published_resource_history = draft.published_resource_history; self.discontinuity_sequence = draft.discontinuity_sequence; @@ -502,7 +508,6 @@ impl HlsSession { /// /// The snapshot contains only metadata and is safe to evaluate after releasing the session lock. pub fn ready_timeline_snapshot(&self, first_proxy_seq: u64, now_ms: u64) -> HlsReadyTimelineSnapshot { - let mut start_ms = 0_u64; let mut expected_proxy_seq = first_proxy_seq; let mut units = Vec::new(); for (&proxy_seq, segment) in self.segments.range(first_proxy_seq..) { @@ -522,7 +527,7 @@ impl HlsSession { }); units.push(HlsReadyTimelineUnit { proxy_seq, - start_ms, + start_ms: segment.media_start_ms, duration_ms: segment.duration_ms, state: if matches!(segment.status, SegmentCacheStatus::Ready { .. }) { HlsReadyMediaState::Ready @@ -533,7 +538,6 @@ impl HlsSession { required_key_ready: segment.encryption.is_none() || key_ready_valid_until_ms.is_some(), key_ready_valid_until_ms, }); - start_ms = start_ms.saturating_add(segment.duration_ms); expected_proxy_seq = expected_proxy_seq.saturating_add(1); } HlsReadyTimelineSnapshot { units: Arc::from(units) } @@ -861,6 +865,13 @@ impl TimelineDraft { effective_host_id: u64, handoff_discontinuity_before: bool, ) -> Result { + let map_ref = parsed + .map_ref + .map(|map_id| { + let parsed_map = manifest.maps.get(map_id).ok_or(TimelineMapError::UnsupportedMapExtension)?; + self.map_origin_map(parsed_map) + }) + .transpose()?; let current_epoch_key = self.origin_segment_key(effective_host_id, parsed.origin_seq); if let Some(proxy_seq) = self.origin_to_proxy.get(¤t_epoch_key).copied() { if let Some(entry) = self.segments.get_mut(&proxy_seq) { @@ -871,6 +882,7 @@ impl TimelineDraft { valid_until_ms: None, }); entry.encryption = map_segment_encryption(parsed)?; + entry.map_ref = map_ref; } return Ok(proxy_seq); } @@ -883,18 +895,12 @@ impl TimelineDraft { let proxy_seq = self.proxy_next_seq.unwrap_or_default(); self.proxy_next_seq = Some(proxy_seq.checked_add(1).ok_or(TimelineMapError::ProxySequenceOverflow)?); - let map_ref = parsed - .map_ref - .map(|map_id| { - let parsed_map = manifest.maps.get(map_id).ok_or(TimelineMapError::UnsupportedMapExtension)?; - self.map_origin_map(parsed_map) - }) - .transpose()?; let proxy_file_ext = proxy_extension_from_url(&parsed.resolved_origin_url, SEGMENT_EXTENSIONS) .ok_or(TimelineMapError::UnsupportedSegmentExtension)?; let entry = SegmentEntry { origin_key, proxy_seq, + media_start_ms: self.next_media_start_ms, duration_ms: parsed.duration_ms, content_type: default_content_type_for_segment_ext(&proxy_file_ext).to_string(), cache_key: SegmentCacheKey::new(self.proxy_session_id.clone(), proxy_seq, &proxy_file_ext), @@ -916,6 +922,7 @@ impl TimelineDraft { access: Arc::new(CacheAccessState::new()), }; + self.next_media_start_ms = self.next_media_start_ms.saturating_add(parsed.duration_ms); self.origin_to_proxy.insert(origin_key, proxy_seq); self.segments.insert(proxy_seq, entry); Ok(proxy_seq) @@ -1901,4 +1908,36 @@ mod tests { assert_eq!(session.segments.get(&0).expect("first segment").map_ref, Some(ProxyMapId(0))); assert_eq!(session.segments.get(&1).expect("second segment").map_ref, Some(ProxyMapId(1))); } + + #[test] + fn media_clock_survives_complete_collection_and_origin_rollover_with_variable_durations() { + let mut session = session(); + session + .apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-MEDIA-SEQUENCE:100\n\ + #EXTINF:4,\n100.ts\n#EXTINF:6,\n101.ts\n#EXTINF:5,\n102.ts\n", + )) + .expect("initial variable timeline"); + assert_eq!( + session.segments.values().map(|entry| entry.media_start_ms).collect::>(), + vec![0, 4_000, 10_000] + ); + session.segments.clear(); + session.origin_to_proxy.clear(); + session + .apply_origin_rebase_manifest( + &normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-MEDIA-SEQUENCE:0\n\ + #EXTINF:6,\nnew-0.ts\n#EXTINF:3,\nnew-1.ts\n", + ), + 2, + false, + ) + .expect("rebased timeline after collection"); + assert_eq!( + session.segments.values().map(|entry| (entry.proxy_seq, entry.media_start_ms)).collect::>(), + vec![(3, 15_000), (4, 21_000)] + ); + assert_eq!(session.next_media_start_ms, 24_000); + } } diff --git a/backend/hls/src/transient.rs b/backend/hls/src/transient.rs index 860cd4b36..cfa397735 100644 --- a/backend/hls/src/transient.rs +++ b/backend/hls/src/transient.rs @@ -123,6 +123,7 @@ pub struct TransientResourceRef { pub expires_at_ms: u64, pub access: Arc, revision: TransientResourceRevision, + origin_account_generation: u64, } #[derive(Clone, Eq, PartialEq)] @@ -161,6 +162,8 @@ pub struct TransientObjectFetchToken { #[derive(Clone)] pub struct TransientPassthroughState { pub resources: HashMap, + invalid_origin_resources: HashSet, + origin_account_generation: u64, pub object_cache: HashMap, object_fetch_notifiers: HashMap>, next_resource_revision: u64, @@ -193,6 +196,8 @@ impl fmt::Debug for TransientPassthroughState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TransientPassthroughState") .field("resources_len", &self.resources.len()) + .field("invalid_origin_resources", &self.invalid_origin_resources.len()) + .field("origin_account_generation", &self.origin_account_generation) .field("object_cache_len", &self.object_cache.len()) .field("object_fetch_notifiers_len", &self.object_fetch_notifiers.len()) .field("next_resource_revision", &self.next_resource_revision) diff --git a/backend/hls/src/transient/identity.rs b/backend/hls/src/transient/identity.rs index 521426ce0..236fe3eb5 100644 --- a/backend/hls/src/transient/identity.rs +++ b/backend/hls/src/transient/identity.rs @@ -282,6 +282,15 @@ impl TransientResourceRef { expires_at_ms: now_ms.saturating_add(ttl_ms), access: Arc::new(CacheAccessState::new()), revision: TransientResourceRevision::default(), + origin_account_generation: 0, + } + } + + pub(crate) fn scope_to_origin_account(&mut self, generation: u64, rewrite_secret: &[u8]) { + self.origin_account_generation = generation; + if generation != 0 { + let identity = format!("account-generation:{generation}\0{}", self.resolved_origin_uri); + self.id = build_transient_resource_id(&identity, rewrite_secret); } } @@ -301,6 +310,7 @@ impl TransientResourceRef { pub(super) fn has_same_cache_identity(&self, other: &Self) -> bool { self.id == other.id + && self.origin_account_generation == other.origin_account_generation && self.kind == other.kind && self.encrypted_media == (self.encrypted_media || other.encrypted_media) && self.resolved_origin_uri == other.resolved_origin_uri @@ -324,6 +334,7 @@ impl fmt::Debug for TransientResourceRef { .field("expires_at_ms", &self.expires_at_ms) .field("active_readers", &self.access.active_readers()) .field("revision", &self.revision) + .field("origin_account_generation", &self.origin_account_generation) .finish() } } diff --git a/backend/hls/src/transient/manifests.rs b/backend/hls/src/transient/manifests.rs index dd82b9495..0f45b2d7b 100644 --- a/backend/hls/src/transient/manifests.rs +++ b/backend/hls/src/transient/manifests.rs @@ -28,6 +28,8 @@ impl TransientPassthroughState { pub fn new(resource_ttl_ms: u64) -> Self { Self { resources: HashMap::new(), + invalid_origin_resources: HashSet::new(), + origin_account_generation: 0, object_cache: HashMap::new(), object_fetch_notifiers: HashMap::new(), next_resource_revision: 1, diff --git a/backend/hls/src/transient/resources.rs b/backend/hls/src/transient/resources.rs index f40303d9c..8dcc72864 100644 --- a/backend/hls/src/transient/resources.rs +++ b/backend/hls/src/transient/resources.rs @@ -6,11 +6,29 @@ use super::{ use std::{collections::HashSet, sync::Arc}; impl TransientPassthroughState { + pub(crate) fn invalidate_origin_resources(&mut self) { + self.invalid_origin_resources.extend(self.resources.keys().cloned()); + self.origin_account_generation = self.origin_account_generation.saturating_add(1); + self.rolling_event_rewrite_seed = None; + } + + pub(crate) fn origin_account_generation(&self) -> u64 { self.origin_account_generation } + + /// Whether this mapping may initiate origin work for the current account. + pub fn origin_resource_is_fetchable(&self, resource: &TransientResourceRef, now_ms: u64) -> bool { + resource.origin_account_generation == self.origin_account_generation + && !self.invalid_origin_resources.contains(&resource.id) + && self.resource_matches_current(resource, now_ms) + } + pub fn upsert_resources(&mut self, resources: I) where I: IntoIterator, { for mut resource in resources { + if resource.origin_account_generation == self.origin_account_generation { + self.invalid_origin_resources.remove(&resource.id); + } let preserve_revision = self .resources .get(&resource.id) @@ -123,6 +141,7 @@ impl TransientPassthroughState { .map(|(id, _)| id.clone()) .collect::>(); self.resources.retain(|id, _| !expired_resource_ids.contains(id)); + self.invalid_origin_resources.retain(|id| !expired_resource_ids.contains(id)); } pub fn active_resource_readers(&self) -> u32 { diff --git a/backend/hls/src/transient/tests/streaming.rs b/backend/hls/src/transient/tests/streaming.rs index d8ff5c2bb..060778768 100644 --- a/backend/hls/src/transient/tests/streaming.rs +++ b/backend/hls/src/transient/tests/streaming.rs @@ -67,3 +67,84 @@ fn expired_finalized_object_is_refetched_instead_of_served_stale() { super::super::TransientObjectFetchDecision::Fetch(_) )); } + +#[test] +fn account_handoff_retains_cached_transient_bytes_but_blocks_old_origin_mapping() { + let mut state = TransientPassthroughState::default(); + let proxy = ProxySessionId("account-handoff".to_string()); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://origin.example/old/media.ts", + b"secret", + 1, + 100, + Some("ts".to_string()), + ); + let id = resource.id.clone(); + state.upsert_resources([resource]); + let resource = state.resolve_current_resource(&id, 2).unwrap(); + let token = match state.begin_object_fetch(&proxy, &resource, "ts", 2, 100) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("uncached media starts one fetch") + } + }; + assert!(state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 3, 100)); + state.invalidate_origin_resources(); + let key = TransientPassthroughState::transient_object_key(&proxy, &id, "ts"); + assert!(state.ready_object(&key, TransientResourceKind::Segment, 4).is_some()); + assert!(state.resolve_current_resource(&id, 4).is_some()); + assert!(!state.origin_resource_is_fetchable(&resource, 4)); + state.upsert_resources([resource.clone()]); + assert!(!state.origin_resource_is_fetchable(&resource, 5)); + assert!(state.ready_object(&key, TransientResourceKind::Segment, 5).is_some()); + state.invalidate_origin_resources(); + state.prune_expired(102); + assert!(state.invalid_origin_resources.is_empty()); +} + +#[test] +fn account_handoff_gives_identical_key_uri_a_new_cache_identity_and_retains_the_old_key() { + let mut state = TransientPassthroughState::default(); + let proxy = ProxySessionId("key-handoff".to_string()); + let body = "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"shared.key\"\n#EXTINF:4,\nmedia.ts\n"; + let old = TransientManifestRewriter::rewrite(body, "https://origin.example/index.m3u8", &proxy, b"secret", 1, 100); + let old_key = old.resources.iter().find(|resource| resource.kind == TransientResourceKind::Key).unwrap().id.clone(); + state.upsert_resources(old.resources); + let old_key_ref = state.resolve_current_resource(&old_key, 2).unwrap(); + let super::super::TransientObjectFetchDecision::Fetch(token) = + state.begin_object_fetch(&proxy, &old_key_ref, "key", 2, 100) + else { + panic!("old key starts its first cache fill") + }; + assert!(state.mark_object_ready_if_current(&token, "application/octet-stream".to_string(), 16, 3, 100)); + state.invalidate_origin_resources(); + let new = TransientManifestRewriter::rewrite_with_options( + body, + "https://origin.example/index.m3u8", + &proxy, + b"secret", + 4, + 100, + crate::transient_manifest::TransientRewriteOptions { + origin_account_generation: state.origin_account_generation(), + ..crate::transient_manifest::TransientRewriteOptions::default() + }, + ); + let new_key = new.resources.iter().find(|resource| resource.kind == TransientResourceKind::Key).unwrap().id.clone(); + assert_ne!(old_key, new_key); + assert_ne!(old.body, new.body); + state.upsert_resources(new.resources); + let new_key_ref = state.resolve_current_resource(&new_key, 5).unwrap(); + assert_eq!(old_key_ref.resolved_origin_uri, new_key_ref.resolved_origin_uri); + let old_lookup = TransientPassthroughState::transient_object_key(&proxy, &old_key, "key"); + let new_lookup = TransientPassthroughState::transient_object_key(&proxy, &new_key, "key"); + assert!(state.ready_object(&old_lookup, TransientResourceKind::Key, 5).is_some()); + assert!(state.ready_object(&new_lookup, TransientResourceKind::Key, 5).is_none()); + assert!(!state.origin_resource_is_fetchable(&old_key_ref, 5)); + assert!(state.origin_resource_is_fetchable(&new_key_ref, 5)); + assert!(matches!( + state.begin_object_fetch(&proxy, &new_key_ref, "key", 5, 100), + super::super::TransientObjectFetchDecision::Fetch(_) + )); +} diff --git a/backend/hls/src/transient_fetcher/tests/lifecycle.rs b/backend/hls/src/transient_fetcher/tests/lifecycle.rs index e628a5335..eadc907bd 100644 --- a/backend/hls/src/transient_fetcher/tests/lifecycle.rs +++ b/backend/hls/src/transient_fetcher/tests/lifecycle.rs @@ -149,7 +149,7 @@ async fn dropping_direct_segment_body_is_client_abort_without_state_transition() assert_eq!(fixture.segment_failure_count().await, 1); assert_eq!(fixture.resource.access.active_readers(), 0); assert_eq!(origin.requests.lock().await.len(), 1); - assert!(take_body_failure_log_attempts().is_empty()); + assert_eq!(take_body_failure_log_attempts().len(), 0); } #[tokio::test] @@ -285,6 +285,9 @@ async fn transient_origin_io_guard_finish_clean_decrements_work_once() { reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: Some(started_gen), + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, }; let guard = HlsTransientOriginIoGuard::new( @@ -328,6 +331,9 @@ async fn transient_origin_io_guard_drop_under_lock_contention_decrements_work() reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: Some(started_gen), + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, }; let guard = HlsTransientOriginIoGuard::new( diff --git a/backend/hls/src/transient_manifest.rs b/backend/hls/src/transient_manifest.rs index b92490087..516b94675 100644 --- a/backend/hls/src/transient_manifest.rs +++ b/backend/hls/src/transient_manifest.rs @@ -72,11 +72,14 @@ pub struct TransientRewriteResult { #[derive(Debug, Clone, Default)] pub struct TransientRewriteCheckpoint { encryption_state: TransientEncryptionState, + origin_account_generation: u64, } #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] pub struct TransientRewriteOptions { pub handoff_discontinuity_sequence: Option, + /// Separates account-dependent resources without changing their concrete origin URL. + pub origin_account_generation: u64, } /// Rewrites only HLS URI surfaces to transient live-HLS proxy resources. @@ -88,6 +91,7 @@ impl TransientManifestRewriter { /// `final_manifest_url` must be the concrete URL of the manifest that was actually fetched. For provider-url /// failover and HTTP redirects this can be the selected mirror or final CDN/origin host. Relative segment, MAP and /// key URIs are resolved against this URL so transient resource fetches keep working after redirects. + #[cfg(test)] pub fn rewrite( body: &str, final_manifest_url: &str, @@ -127,7 +131,10 @@ impl TransientManifestRewriter { reverse_proxy_rewrite_secret, now_ms, ttl_ms, - TransientRewriteCheckpoint::default(), + TransientRewriteCheckpoint { + origin_account_generation: options.origin_account_generation, + ..TransientRewriteCheckpoint::default() + }, ); if let Some(discontinuity_sequence) = options.handoff_discontinuity_sequence { result.body = apply_handoff_discontinuity_boundary(&result.body, discontinuity_sequence); @@ -178,6 +185,7 @@ fn rewrite_transient_fragment( now_ms, ttl_ms, resources: &mut resources, + origin_account_generation: checkpoint.origin_account_generation, }; for part in body.split_inclusive('\n') { processed_line_count = processed_line_count.saturating_add(1); @@ -340,6 +348,7 @@ struct TransientResourceRewriteContext<'a> { now_ms: u64, ttl_ms: u64, resources: &'a mut HashMap, + origin_account_generation: u64, } impl TransientResourceRewriteContext<'_> { @@ -396,6 +405,7 @@ impl TransientResourceRewriteContext<'_> { self.ttl_ms, Some(extension.clone()), ); + resource.scope_to_origin_account(self.origin_account_generation, self.reverse_proxy_rewrite_secret); resource.encrypted_media = encrypted_media; let resource_id = resource.id.clone(); self.resources @@ -780,7 +790,7 @@ mod tests { b"secret", 100, 1_000, - TransientRewriteOptions { handoff_discontinuity_sequence: Some(0) }, + TransientRewriteOptions { handoff_discontinuity_sequence: Some(0), ..TransientRewriteOptions::default() }, ) } diff --git a/backend/iptv/src/stalker/cookie_jar.rs b/backend/iptv/src/stalker/cookie_jar.rs index aad820778..06dec6408 100644 --- a/backend/iptv/src/stalker/cookie_jar.rs +++ b/backend/iptv/src/stalker/cookie_jar.rs @@ -250,6 +250,6 @@ mod tests { let jar = StalkerCookieJar::new(); jar.ingest_set_cookie(["stb_lang=en; Max-Age=10"], 1_000); assert!(jar.active_cookie_header(1_009).contains("stb_lang=en")); - assert!(jar.active_cookie_header(1_010).is_empty()); + assert_eq!(jar.active_cookie_header(1_010).len(), 0); } } diff --git a/backend/iptv/src/xtream/mod.rs b/backend/iptv/src/xtream/mod.rs index 1a1d35857..c4447c929 100644 --- a/backend/iptv/src/xtream/mod.rs +++ b/backend/iptv/src/xtream/mod.rs @@ -1038,14 +1038,14 @@ mod tests { let sink = RecordingSink::default(); // One second before the window opens. super::notify_account_expire_at(EXPIRY - super::THREE_DAYS_IN_SECS - 1, Some(EXPIRY), &sink, "u", "p"); - assert!(sink.states().is_empty()); + assert_eq!(sink.states().len(), 0); } #[test] fn no_expiry_date_emits_nothing() { let sink = RecordingSink::default(); super::notify_account_expire_at(EXPIRY, None, &sink, "user", "provider"); - assert!(sink.states().is_empty()); + assert_eq!(sink.states().len(), 0); } fn options_with_flags(flags: &[ConfigInputFlags]) -> ConfigInputOptions { @@ -1173,7 +1173,7 @@ mod tests { assert_eq!(baseline_reads.load(Ordering::SeqCst), 0); assert_eq!(direct_fetch.groups.len(), 1); assert_eq!(direct_fetch.groups[0].channels.len(), 89); - assert!(direct_fetch.quality_rejections.is_empty()); + assert_eq!(direct_fetch.quality_rejections.len(), 0); let mut update_fetch = PlaylistFetch::default(); apply_in_memory_cluster_download( @@ -1241,7 +1241,7 @@ mod tests { .expect("deduplicated candidate should be accepted"); assert_eq!(baseline_reads.load(Ordering::SeqCst), 1); - assert!(fetch.quality_rejections.is_empty()); + assert_eq!(fetch.quality_rejections.len(), 0); assert_eq!( fetch.quality_acceptances, vec![ClusterUpdateAcceptance { @@ -1319,8 +1319,8 @@ mod tests { assert_eq!(baseline_reads.load(Ordering::SeqCst), 0); assert_eq!(direct_fetch.groups.iter().map(|group| group.channels.len()).sum::(), 2); assert_eq!(disabled_fetch.groups.iter().map(|group| group.channels.len()).sum::(), 2); - assert!(direct_fetch.quality_rejections.is_empty()); - assert!(disabled_fetch.quality_rejections.is_empty()); + assert_eq!(direct_fetch.quality_rejections.len(), 0); + assert_eq!(disabled_fetch.quality_rejections.len(), 0); } #[tokio::test] @@ -1359,7 +1359,7 @@ mod tests { .expect("forced candidate should be accepted"); assert_eq!(baseline_reads.load(Ordering::SeqCst), 0); - assert!(fetch.quality_rejections.is_empty()); + assert_eq!(fetch.quality_rejections.len(), 0); assert_eq!(fetch.groups.len(), 1); assert_eq!(fetch.groups[0].channels.len(), 2); assert_eq!(fetch.groups[0].channels[0].header.name.as_ref(), "winning-duplicate"); @@ -1409,7 +1409,7 @@ mod tests { vec![candidate_group(XtreamCluster::Live, 1, 1)], ); assert_eq!(disabled.groups[0].channels.len(), 1); - assert!(disabled.quality_rejections.is_empty()); + assert_eq!(disabled.quality_rejections.len(), 0); let mut bootstrap = PlaylistFetch::default(); apply_cluster_update_quality( @@ -1420,7 +1420,7 @@ mod tests { vec![candidate_group(XtreamCluster::Video, 2, 3)], ); assert_eq!(bootstrap.groups[0].channels.len(), 3); - assert!(bootstrap.quality_rejections.is_empty()); + assert_eq!(bootstrap.quality_rejections.len(), 0); let mut empty_bootstrap = PlaylistFetch::default(); apply_cluster_update_quality(&mut empty_bootstrap, XtreamCluster::Series, None, 90, Vec::new()); diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs index 1df0ebae8..9415954f3 100644 --- a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs @@ -12,7 +12,7 @@ fn pipeline_transparency_telemetry_reports_non_cluster_cache_without_synthetic_c assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); assert_eq!(telemetry.source, Some(PlaylistUpdateDataSource::Cache)); - assert!(telemetry.clusters.is_empty()); + assert_eq!(telemetry.clusters.len(), 0); } #[test] diff --git a/backend/processing/src/processor/playlist/tests/curation_publication.rs b/backend/processing/src/processor/playlist/tests/curation_publication.rs index ad84eac27..84d9b3a54 100644 --- a/backend/processing/src/processor/playlist/tests/curation_publication.rs +++ b/backend/processing/src/processor/playlist/tests/curation_publication.rs @@ -342,7 +342,7 @@ impl Publication { .all(|item| text.contains(item.header.title.as_ref()) && text.contains(item.header.url.as_ref())) })); let strm = self.strm_contents(); - assert!(!strm.is_empty()); + assert_ne!(strm.len(), 0); let files = file_snapshot(self.directory.path()); let live_title = expected[0].channels[0].header.title.as_bytes(); assert!( @@ -352,7 +352,7 @@ impl Publication { }), "new EPG programme must be published" ); - assert!(!self.cache_signature().await.is_empty()); + assert_ne!(self.cache_signature().await.len(), 0); } async fn identity_signature(&self) -> BTreeMap { @@ -455,7 +455,7 @@ async fn tmdb_https_selection_only_publishes_m3u_strm_without_an_xtream_output() let text = std::fs::read_to_string(run.directory.path().join("published.m3u")).unwrap(); assert!(text.contains("First") && text.contains("Show S01E01") && text.contains("Live")); assert!(!text.contains("Unselected") && !text.contains("TMDB")); - assert!(!run.strm_contents().is_empty()); + assert_ne!(run.strm_contents().len(), 0); let cache = run.context.playlist_state.as_ref().unwrap().data.read().await; assert!(cache[&run.target.name].xtream.is_none()); assert_eq!(server.requests.lock().unwrap().len(), 3); @@ -469,7 +469,7 @@ async fn tmdb_https_empty_and_no_match_clear_published_vod_series_but_keep_live( let server = DiscoveryServer::start().await; let run = Publication::new(&server, &target("curated", false, true)); run.publish().await.unwrap(); - assert!(!run.strm_contents().is_empty()); + assert_ne!(run.strm_contents().len(), 0); server.reply(MOVIES, 200, body); server.reply(TV, 200, body); run.publish().await.unwrap(); diff --git a/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs index bf9907230..c9dae7600 100644 --- a/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs +++ b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs @@ -248,7 +248,7 @@ async fn curation_production_wiring_does_not_fall_back_to_the_generic_client() { server.requests.lock().unwrap().is_empty(), "generic client has the fixture CA and could succeed, but is not used" ); - assert!(!run.cache_signature().await.is_empty()); + assert_ne!(run.cache_signature().await.len(), 0); assert!(!file_snapshot(run.directory.path()).is_empty()); } @@ -272,7 +272,7 @@ async fn curation_failure_respects_empty_input_authorization() { assert_eq!(result.is_err(), authorized, "ordinary M3U persistence still rejects an empty playlist"); assert_eq!(server.requests.lock().unwrap().len(), if authorized { 3 } else { 0 }); assert!(file_snapshot(run.directory.path()).is_empty()); - assert!(run.cache_signature().await.is_empty()); + assert_eq!(run.cache_signature().await.len(), 0); } } diff --git a/backend/processing/src/processor/playlist/tests/mapping_stage.rs b/backend/processing/src/processor/playlist/tests/mapping_stage.rs index 395391c8b..dc47f1fb7 100644 --- a/backend/processing/src/processor/playlist/tests/mapping_stage.rs +++ b/backend/processing/src/processor/playlist/tests/mapping_stage.rs @@ -151,7 +151,7 @@ async fn m3u_alias_playlist_is_downloaded_and_indexed_separately() { assert!(errors.is_empty(), "unexpected download errors: {errors:?}"); assert!(storage_error.is_none(), "unexpected primary storage error: {storage_error:?}"); assert!(!partial); - assert!(quality_rejections.is_empty()); + assert_eq!(quality_rejections.len(), 0); assert!(!primary_playlist.is_empty()); let alias_url = tuliprox_repository::load_input_m3u_stream_url( &ctx.config, @@ -212,7 +212,7 @@ async fn failed_m3u_alias_is_retried_after_primary_input_is_processed() { assert!(!first_errors.is_empty(), "missing alias should report an error"); assert!(first_storage_error.is_none(), "primary storage should succeed: {first_storage_error:?}"); assert!(!first_partial); - assert!(first_quality_rejections.is_empty()); + assert_eq!(first_quality_rejections.len(), 0); assert!(!first_playlist.is_empty()); assert!(ctx.is_input_downloaded("main-account").await); assert!(!ctx.is_input_downloaded("retry-account").await); @@ -236,7 +236,7 @@ async fn failed_m3u_alias_is_retried_after_primary_input_is_processed() { assert!(second_errors.is_empty(), "unexpected retry errors: {second_errors:?}"); assert!(second_storage_error.is_none(), "primary storage should remain readable: {second_storage_error:?}"); assert!(!second_partial); - assert!(second_quality_rejections.is_empty()); + assert_eq!(second_quality_rejections.len(), 0); assert!(!second_playlist.is_empty()); assert!(ctx.is_input_downloaded("retry-account").await); let alias_url = tuliprox_repository::load_input_m3u_stream_url( diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs index 47c302c4b..62ec79d31 100644 --- a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs @@ -313,8 +313,8 @@ async fn playlist_update_run_manual_and_automatic_execution_order_is_generated_a }; let manual_progress = progress_for(&manual_run_id); let automatic_progress = progress_for(&automatic_run_id); - assert!(!manual_progress.is_empty()); - assert!(!automatic_progress.is_empty()); + assert_ne!(manual_progress.len(), 0); + assert_ne!(automatic_progress.len(), 0); assert!(manual_progress.iter().all(|progress| progress.execution_order == Some(manual_order))); assert!(automatic_progress.iter().all(|progress| progress.execution_order == Some(automatic_order))); } @@ -687,11 +687,11 @@ async fn coderabbit_stalker_global_failures_mark_only_requested_non_skipped_clus assert_eq!(fetch.errors.len(), 1, "{failure:?}"); assert!(!fetch.is_ok()); assert!(fetch.quality_acceptances.is_empty() && fetch.quality_rejections.is_empty()); - assert!(fetch.force_updates.is_empty()); + assert_eq!(fetch.force_updates.len(), 0); let requests = server.finish(); match failure { - FailurePoint::Handshake => assert!(!requests.is_empty()), - FailurePoint::Portal | FailurePoint::Client | FailurePoint::Storage => assert!(requests.is_empty()), + FailurePoint::Handshake => assert_ne!(requests.len(), 0), + FailurePoint::Portal | FailurePoint::Client | FailurePoint::Storage => assert_eq!(requests.len(), 0), } } } @@ -725,7 +725,7 @@ async fn coderabbit_xtream_http_failure_preserves_successful_disk_clusters() { assert_eq!(fetch.failed_clusters, vec![XtreamCluster::Video]); assert!(!fetch.is_ok(), "the failed cluster must remain a technical failure"); assert!(fetch.persisted); - assert!(fetch.quality_rejections.is_empty()); + assert_eq!(fetch.quality_rejections.len(), 0); let accepted: Vec<_> = match quality { UpdateQualityPolicy::Enforce => fetch.quality_acceptances.iter().map(|value| value.cluster).collect(), UpdateQualityPolicy::Bypass => fetch.force_updates.iter().map(|value| value.cluster).collect(), @@ -805,7 +805,7 @@ async fn pipeline_transparency_normal_policy_reuses_a_valid_cluster_cache() { assert!(result.errors.is_empty()); assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert_eq!(result.job_state(), InputJobState::Ready); assert_baseline(&result.source.take_groups()); assert!(requests.is_empty(), "valid cache should avoid provider requests: {requests:?}"); @@ -1215,7 +1215,7 @@ async fn force_policy_retains_a_cluster_after_a_technical_parse_failure() { assert!(!result.errors.is_empty(), "malformed VOD response must remain a technical error"); assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let groups = result.source.take_groups(); let vod = groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).expect("retained VOD"); assert_eq!(vod.title.as_ref(), "old-vod"); @@ -1278,7 +1278,7 @@ async fn in_memory_series_quality_compares_catalog_rows_with_an_enriched_baselin let requests = server.finish(); assert!(result.errors.is_empty(), "Series update failed: {:?}", result.errors); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let groups = result.source.take_groups(); assert_eq!(groups.len(), 1); assert_eq!(groups[0].title.as_ref(), "candidate-series"); @@ -1388,7 +1388,7 @@ async fn in_memory_quality_accepts_unique_population_and_persists_the_last_dupli assert!(result.errors.is_empty(), "deduplicated acceptance failed: {:?}", result.errors); assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let groups = result.source.take_groups(); assert_eq!(groups.iter().map(|group| group.channels.len()).sum::(), 100); let winner_group = groups.iter().find(|group| group.id == 2).expect("winning category"); @@ -1443,7 +1443,7 @@ async fn m3u_update_quality_enforces_the_existing_cluster_threshold() { let mut initial = download_input(&initial_ctx, &input, false).await; assert!(initial.errors.is_empty(), "initial M3U download failed: {:?}", initial.errors); - assert!(initial.quality_rejections.is_empty()); + assert_eq!(initial.quality_rejections.len(), 0); assert_eq!(initial.source.get_channel_count(), 2); tokio::fs::write(&playlist_path, "#EXTM3U\n#EXTINF:-1,New\nhttp://new.example/1\n") diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs index 7e93b080d..eaca5ceef 100644 --- a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs @@ -105,7 +105,7 @@ async fn m3u_update_quality_all_clusters_accepted() { let temp = tempfile::tempdir().unwrap(); let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; let mut result = run(&ctx, &input, [9, 11, 10]).await; - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert_eq!(result.update_state(), PlaylistUpdateState::Success); verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await; } @@ -257,7 +257,7 @@ async fn m3u_update_quality_disabled() { let temp = tempfile::tempdir().unwrap(); let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [0, 0, 0]).await; let mut result = run(&ctx, &input, [1, 1, 0]).await; - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert!(result .input_telemetry .as_ref() @@ -273,7 +273,7 @@ async fn m3u_update_quality_first_import() { let temp = tempfile::tempdir().unwrap(); let (ctx, input) = fixture(temp.path(), None, [90, 90, 90]).await; let mut result = run(&ctx, &input, [1, 1, 1]).await; - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert!(result.input_telemetry.as_ref().unwrap().clusters.iter().all(|c| c.quality.is_none() && c.baseline_count.is_none() && c.decision == Some(PlaylistUpdateClusterDecision::Accepted))); @@ -292,7 +292,7 @@ async fn m3u_update_quality_download_or_parse_failure() { } let mut result = download_input(&ctx, &input, false).await; assert!(!result.errors.is_empty()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert_eq!(result.update_state(), PlaylistUpdateState::Failure); let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); @@ -324,7 +324,7 @@ async fn m3u_update_quality_skip_preserves_previous_and_reuse_emits_no_new_facts Arc::make_mut(&mut input).options.as_mut().unwrap().flags.set(ConfigInputFlags::SkipSeries); let mut result = run(&ctx, &input, [10, 10, 0]).await; assert_eq!(result.update_state(), PlaylistUpdateState::Success); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); assert_eq!(counts(&result.source.take_groups()), [10, 10, 0]); assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); @@ -388,7 +388,7 @@ async fn m3u_update_quality_force_keeps_parse_and_empty_publication_errors() { tokio::fs::write(&input.url, body).await.unwrap(); let mut result = download_input(&ctx, &input, false).await; assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); assert_eq!(counts(&saved.take_groups()), [10; 3]); } diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs index 0d2bd45fe..39db5fba2 100644 --- a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs @@ -93,7 +93,7 @@ async fn m3u_update_quality_playlist_update_status_staged_completion_is_independ let telemetry = completions[0].input_telemetry.as_ref().unwrap(); assert_eq!(telemetry.source, None); assert!(telemetry.clusters.iter().all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider))); - assert!(staged_completion::input_completions(&events, parent.id).is_empty()); + assert_eq!(staged_completion::input_completions(&events, parent.id).len(), 0); } } @@ -243,7 +243,7 @@ async fn playlist_update_status_parent_download_persists_staged_own_per_cluster_ let skipped = download_input(&ctx, &staged, false).await; assert!(skipped.input_telemetry.is_none()); assert!(input_cache::load_input_status(&staged_path).last_input_update.is_none()); - assert!(staged_completion::input_completions(&events, staged.id).is_empty()); + assert_eq!(staged_completion::input_completions(&events, staged.id).len(), 0); let result = download_input(&ctx, &parent, false).await; let status = input_cache::load_input_status(&staged_path); diff --git a/backend/processing/src/processor/stalker.rs b/backend/processing/src/processor/stalker.rs index e4edc956a..f5fe926a2 100644 --- a/backend/processing/src/processor/stalker.rs +++ b/backend/processing/src/processor/stalker.rs @@ -667,7 +667,7 @@ mod tests { fn playback_candidates_are_empty_without_any_command() { let item = stalker_item("", None); - assert!(playback_candidates(&item).is_empty()); + assert_eq!(playback_candidates(&item).len(), 0); } #[test] diff --git a/backend/processing/src/processor/stalker_refresh.rs b/backend/processing/src/processor/stalker_refresh.rs index 5009cc3f3..46c236e68 100644 --- a/backend/processing/src/processor/stalker_refresh.rs +++ b/backend/processing/src/processor/stalker_refresh.rs @@ -1314,7 +1314,7 @@ mod tests { finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(0, 0, 100)) .await?; - assert!(rejections.is_empty()); + assert_eq!(rejections.len(), 0); let published = load_active_manifest(temp.path(), identity).await?; assert_eq!(published.series.as_ref().map(|files| files.generation), Some(generation)); Ok(()) @@ -1377,7 +1377,7 @@ mod tests { ) .await?; - assert!(reports.quality_rejections.is_empty()); + assert_eq!(reports.quality_rejections.len(), 0); assert_eq!( reports.force_updates, vec![ClusterForceUpdate { cluster: XtreamCluster::Video, candidate_count: 0, configured_threshold: 100 }] @@ -1421,7 +1421,7 @@ mod tests { ) .await?; - assert!(reports.quality_rejections.is_empty()); + assert_eq!(reports.quality_rejections.len(), 0); assert_eq!(reports.force_updates.len(), 1); assert_eq!(reports.force_updates[0].cluster, XtreamCluster::Live); let published = load_active_manifest(temp.path(), identity).await?; diff --git a/backend/repository/src/alias_repository.rs b/backend/repository/src/alias_repository.rs index 091c75f5d..57a3b0fb3 100644 --- a/backend/repository/src/alias_repository.rs +++ b/backend/repository/src/alias_repository.rs @@ -1302,7 +1302,7 @@ url;other;other;http://third/get.php?username=user&password=pass;0;2;\n"; ) .await?; assert!(!receipt.changed); - assert!(receipt.matched.is_empty()); + assert_eq!(receipt.matched.len(), 0); assert_eq!(tokio::fs::read_to_string(&path).await?, original); Ok(()) } @@ -1397,7 +1397,7 @@ input_1;user;password;http://provider.tv;1;0;keep-me;old-enabled;old-expiry;trai .await?; assert!(!changed); - assert!(updated.is_empty()); + assert_eq!(updated.len(), 0); assert!(!backup_dir.exists()); Ok(()) } diff --git a/backend/repository/src/lineup_guide_index.rs b/backend/repository/src/lineup_guide_index.rs index 539f1634d..4f39477dd 100644 --- a/backend/repository/src/lineup_guide_index.rs +++ b/backend/repository/src/lineup_guide_index.rs @@ -485,7 +485,7 @@ mod tests { let playlist = vec![group(1, XtreamCluster::Series, vec![series])]; // `m3u_write_playlist` does not store series info, so the index must not point at it. - assert!(entries(&build_lineup_guide_index(&playlist, LineupGuideView::M3u)).is_empty()); + assert_eq!(entries(&build_lineup_guide_index(&playlist, LineupGuideView::M3u)).len(), 0); } #[test] @@ -529,7 +529,7 @@ mod tests { // `lineup.json` publishes the empty EPG id as the guide number of the first channel, which // no request can ask for, so neither the empty id nor the name may answer for it. - assert!(entries(&tree).is_empty()); + assert_eq!(entries(&tree).len(), 0); } #[tokio::test] diff --git a/backend/repository/src/playlist_repository/tests/lifecycle.rs b/backend/repository/src/playlist_repository/tests/lifecycle.rs index 355e9b3f4..24df6fb09 100644 --- a/backend/repository/src/playlist_repository/tests/lifecycle.rs +++ b/backend/repository/src/playlist_repository/tests/lifecycle.rs @@ -54,5 +54,5 @@ async fn intentional_empty_strm_cleanup_false_removes_indexed_files_but_keeps_un get_target_storage_path(&config, &target.name).expect("target storage") }; let index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm-retained")), &target_storage); - assert!(std::fs::read_to_string(index).expect("empty STRM index").is_empty()); + assert_eq!(std::fs::read_to_string(index).expect("empty STRM index").len(), 0); } diff --git a/backend/repository/src/playlist_repository/tests/transport.rs b/backend/repository/src/playlist_repository/tests/transport.rs index 7b4837dd1..90f3bc745 100644 --- a/backend/repository/src/playlist_repository/tests/transport.rs +++ b/backend/repository/src/playlist_repository/tests/transport.rs @@ -102,7 +102,7 @@ async fn intentional_empty_output_views_clear_managed_artifacts_without_alias_le "[]" ); let strm_index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm")), &target_storage); - assert!(std::fs::read_to_string(strm_index).expect("empty STRM index").is_empty()); + assert_eq!(std::fs::read_to_string(strm_index).expect("empty STRM index").len(), 0); { let cache = playlist_state.data.read().await; let cached = cache.get(&target.name).expect("empty target cache"); diff --git a/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs b/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs index b320a265a..b5ae4fed0 100644 --- a/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs +++ b/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs @@ -190,8 +190,8 @@ async fn disk_force_keeps_active_files_after_a_technical_staging_failure() { .await; assert_eq!(result.errors.len(), 1); - assert!(result.outcomes.is_empty()); - assert!(result.force_updates.is_empty()); + assert_eq!(result.outcomes.len(), 0); + assert_eq!(result.force_updates.len(), 0); assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); assert_no_refresh_artifacts(&storage_path); } @@ -233,7 +233,7 @@ async fn pipeline_transparency_disk_quality_survives_post_evaluation_staging_fai quality: Some(95), }] ); - assert!(result.outcomes.is_empty()); + assert_eq!(result.outcomes.len(), 0); assert_eq!(result.errors.len(), 1); assert_eq!(result.failed_cluster, Some(XtreamCluster::Live)); assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); diff --git a/backend/session/src/active_user_manager/connections.rs b/backend/session/src/active_user_manager/connections.rs index a44a5a026..09627c6c0 100644 --- a/backend/session/src/active_user_manager/connections.rs +++ b/backend/session/src/active_user_manager/connections.rs @@ -12,10 +12,7 @@ use arc_swap::ArcSwapOption; use lru::LruCache; use shared::{ defaults::{default_grace_period_millis, default_grace_period_timeout_secs}, - model::{ - ActiveUserConnectionChange, CustomVideoStreamType, EventMessage, StreamChannel, StreamInfo, - UserConnectionPermission, VirtualId, - }, + model::{CustomVideoStreamType, StreamChannel, StreamInfo, UserConnectionPermission, VirtualId}, utils::{current_time_secs, sanitize_sensitive_info, strip_port, Internable}, }; use std::{ @@ -690,8 +687,7 @@ impl ActiveUserManager { } if let Some(stream_info) = preserved_update.as_ref() { - self.event_manager - .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); + self.emit_current_stream_update(stream_info).await; } if connection_changed { @@ -870,7 +866,7 @@ impl ActiveUserManager { } for stream_info in preserved_updates { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.emit_current_stream_update(&stream_info).await; } for username in &disconnected_users { diff --git a/backend/session/src/active_user_manager/mod.rs b/backend/session/src/active_user_manager/mod.rs index 97149690a..6ca6d72af 100644 --- a/backend/session/src/active_user_manager/mod.rs +++ b/backend/session/src/active_user_manager/mod.rs @@ -164,7 +164,7 @@ pub enum StreamRequestDetach { } /// How long an ended session token blocks its recreation from a sealed playlist token. -const ENDED_SESSION_RECREATE_BLOCK: Duration = Duration::from_secs(600); +const ENDED_SESSION_RECREATE_BLOCK: Duration = Duration::from_mins(10); pub struct ReleasedConnection { pub addr_removed: bool, @@ -256,7 +256,6 @@ mod session_lifecycle; use self::observability::divergence_key; #[cfg(test)] use self::observability::DivergenceKind; -pub use self::session_lifecycle::next_session_incarnation; use self::{ connections::{ create_socket_reentry_guard_key, decide_connection_kind, url_host_key, PromotionAction, RecentWinnerProtection, @@ -270,3 +269,4 @@ use self::{ uses_session_reentry_guard, UserSessionParams, }, }; +pub use self::{observability::SharedStreamProviderProjection, session_lifecycle::next_session_incarnation}; diff --git a/backend/session/src/active_user_manager/observability.rs b/backend/session/src/active_user_manager/observability.rs index a3326734c..e17e1d206 100644 --- a/backend/session/src/active_user_manager/observability.rs +++ b/backend/session/src/active_user_manager/observability.rs @@ -13,6 +13,34 @@ use std::{ }; use tuliprox_core::utils::debug_if_enabled; +/// Locks viewer rows before an origin publication, so publication and its UI events have no cancellation gap. +pub struct SharedStreamProviderProjection<'a> { + connections: super::UserConnectionsWriteGuard<'a>, + events: &'a EventManager, + shared_stream_id: u64, +} + +impl SharedStreamProviderProjection<'_> { + /// Changes only display metadata and publishes each full changed row while updates are ordered by the lock. + pub fn update_provider(&mut self, provider: &Arc) -> usize { + let mut changed = 0; + for data in self.connections.by_key.values_mut() { + for stream in &mut data.streams { + if stream.channel.shared + && stream.channel.shared_stream_id == Some(self.shared_stream_id) + && stream.provider != *provider + { + stream.provider.clone_from(provider); + self.events + .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream.clone()))); + changed += 1; + } + } + } + changed + } +} + pub(super) struct DivergenceEntry { pub(super) last_logged: Instant, pub(super) count_since_last_log: u32, @@ -44,6 +72,18 @@ pub(super) struct DivergenceSnapshot { } impl ActiveUserManager { + /// Acquire before the HLS session write lock; do not call other manager methods while this guard is held. + pub async fn lock_shared_stream_provider_projection( + &self, + shared_stream_id: u64, + ) -> SharedStreamProviderProjection<'_> { + SharedStreamProviderProjection { + connections: self.write_connections().await, + events: &self.event_manager, + shared_stream_id, + } + } + pub(super) fn custom_stream_technical_info() -> StreamTechnicalInfo { StreamTechnicalInfo { container: String::from("mpegts"), @@ -107,6 +147,23 @@ impl ActiveUserManager { } } + /// Re-read an existing playback row and keep the connection lock through its synchronous event send. + /// A delayed producer must neither restore an older projection nor recreate a removed playback. + pub(crate) async fn emit_current_stream_update(&self, snapshot: &StreamInfo) { + let connections = self.connections.read().await; + let current = connections.by_key.get(&snapshot.username).and_then(|data| { + data.streams.iter().find(|stream| { + stream.uid == snapshot.uid + && stream.session_token == snapshot.session_token + && stream.channel.shared_stream_id == snapshot.channel.shared_stream_id + }) + }); + if let Some(current) = current { + self.event_manager + .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(current.clone()))); + } + } + pub(super) async fn emit_promotion_update(&self, username: &str, action: PromotionAction) { let maybe_stream = { let user_connections = self.connections.read().await; @@ -131,7 +188,7 @@ impl ActiveUserManager { ); } } - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.emit_current_stream_update(&stream_info).await; } } diff --git a/backend/session/src/active_user_manager/tests.rs b/backend/session/src/active_user_manager/tests.rs index 8cd943389..e6d945e9d 100644 --- a/backend/session/src/active_user_manager/tests.rs +++ b/backend/session/src/active_user_manager/tests.rs @@ -4,6 +4,7 @@ mod adaptive_streams; mod admission; mod expiry_gc; mod observability; +mod provider_handover; mod provider_headers; mod reentry_protection; mod request_claims; diff --git a/backend/session/src/active_user_manager/tests/adaptive_streams.rs b/backend/session/src/active_user_manager/tests/adaptive_streams.rs index 0d045f46f..45a93d54c 100644 --- a/backend/session/src/active_user_manager/tests/adaptive_streams.rs +++ b/backend/session/src/active_user_manager/tests/adaptive_streams.rs @@ -200,7 +200,7 @@ async fn test_kicked_release_does_not_preserve_adaptive_stream() { let removed = manager.release_connection_as_kicked(&addr).await; assert!(removed.addr_removed); assert_eq!(removed.removed_streams.len(), 1); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -323,7 +323,7 @@ async fn test_adaptive_session_release_connection_preserves_logical_stream_and_s assert!(released.removed_streams.is_empty(), "adaptive session should remain logically active"); assert_eq!(manager.user_connections("user1").await, 0); assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); let connections = manager.connections.read().await; let preserved_stream = connections @@ -413,7 +413,7 @@ async fn test_release_stream_ignores_preserved_adaptive_entry() { let released = manager.release_connection(&addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); + assert_eq!(released.removed_streams.len(), 0); assert!(manager.release_stream(&addr).await.is_none()); } @@ -448,7 +448,7 @@ async fn test_release_stream_without_session_removes_adaptive_stream_instead_of_ let released = manager.release_stream(&addr).await; assert!(released.is_some(), "stream without schedulable expiry must be removed"); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -597,7 +597,7 @@ async fn catchup_release_connection_preserves_logical_stream_until_session_expir ); assert_eq!(manager.user_connections(&user.username).await, 0); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); let connections = manager.connections.read().await; let preserved_stream = connections @@ -895,7 +895,7 @@ async fn preserved_soft_reactivation_and_cleanup_leave_normal_slot_unchanged() { manager.release_stream_by_uid(&normal_addr, normal_stream_uid).await.expect("normal stream should release"); let connections = manager.connections.read().await; let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert!(connection_data.streams.is_empty()); + assert_eq!(connection_data.streams.len(), 0); assert!(connection_data.stream_kinds.is_empty()); assert_no_real_connection_slots(connection_data); drop(connections); @@ -1105,7 +1105,7 @@ async fn release_session_streams_notifies_frontend_for_each_removed_row() { vec![(addr, 9101)], "lease-idle release removes the panel row and must tell the frontend" ); - assert!(manager.panel_streams().await.is_empty()); + assert_eq!(manager.panel_streams().await.len(), 0); } #[tokio::test] @@ -1156,7 +1156,7 @@ async fn expire_grace_active_notifies_frontend_for_each_removed_row() { manager.expire_grace_active(&user.username, &token, version).await; assert_eq!(drain_disconnected_streams(&mut events), vec![(addr, 9104)]); - assert!(manager.panel_streams().await.is_empty()); + assert_eq!(manager.panel_streams().await.len(), 0); } #[tokio::test] @@ -1218,7 +1218,7 @@ async fn expire_grace_active_removes_every_row_of_the_expired_session() { let mut removed = drain_disconnected_streams(&mut events); removed.sort_unstable_by_key(|(_, uid)| *uid); assert_eq!(removed, vec![(addr, 9107), (second_addr, 9108)]); - assert!(manager.panel_streams().await.is_empty()); + assert_eq!(manager.panel_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/session/src/active_user_manager/tests/expiry_gc.rs b/backend/session/src/active_user_manager/tests/expiry_gc.rs index db3637605..f043b9a5d 100644 --- a/backend/session/src/active_user_manager/tests/expiry_gc.rs +++ b/backend/session/src/active_user_manager/tests/expiry_gc.rs @@ -145,7 +145,7 @@ async fn test_preserved_adaptive_stream_is_pruned_after_session_ttl() { manager .process_due_adaptive_expiry_entries(current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1)) .await; - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -389,7 +389,7 @@ async fn test_due_adaptive_expiry_reschedules_when_session_timestamp_changes() { *expiry_index.get(&key).unwrap() }; assert!(new_expires_at > old_expires_at); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -533,7 +533,7 @@ async fn test_due_adaptive_expiry_does_not_block_on_full_cleanup_channel() { let queued_event = cleanup_rx.try_recv().expect("prefilled cleanup event should remain queued"); assert!(matches!(queued_event, CleanupEvent::ReleaseConnection { .. })); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/session/src/active_user_manager/tests/provider_handover.rs b/backend/session/src/active_user_manager/tests/provider_handover.rs new file mode 100644 index 000000000..0c0d7cad9 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/provider_handover.rs @@ -0,0 +1,321 @@ +use super::{ + test_adaptive_channel, test_user_credentials, ActiveUserConnectionParams, ActiveUserManager, + CreateUserSessionParams, +}; +use crate::{ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{ActiveUserConnectionChange, EventMessage, StreamInfo, UserConnectionPermission}, + utils::Internable, +}; +use std::{borrow::Cow, sync::Arc}; +use tuliprox_core::model::{Config, Fingerprint}; + +async fn viewer(manager: &ActiveUserManager, uid: u32, username: &str, shared_id: Option) -> StreamInfo { + let addr = format!("127.0.0.1:{}", 14000 + uid).parse().unwrap(); + let fingerprint = Fingerprint::new(format!("viewer-{uid}"), "127.0.0.1".into(), addr); + let user = test_user_credentials(username, 10, 0); + let token = format!("session-{uid}"); + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: &token, + virtual_id: uid, + provider: "account-a", + stream_url: "http://origin.test/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager.add_connection(&addr).await; + let mut channel = test_adaptive_channel(uid); + channel.shared = shared_id.is_some(); + channel.shared_stream_id = shared_id; + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: uid + 100, + username, + max_connections: 10, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "account-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("handover-test"), + session_token: Some(&token), + }) + .await + .expect("viewer registers") +} + +async fn fixture() -> Arc { + let events = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveUserManager::new(&Config::default(), &Arc::new(ArcSwapOption::default()), &events)); + viewer(&manager, 1, "alice", Some(42)).await; + viewer(&manager, 2, "bob", Some(42)).await; + viewer(&manager, 3, "alice", Some(42)).await; + manager +} + +async fn project(manager: &ActiveUserManager) -> usize { + manager.lock_shared_stream_provider_projection(42).await.update_provider(&"account-b".intern()) +} + +#[tokio::test] +async fn handover_updates_preserved_viewer_rows() { + let manager = fixture().await; + manager.release_connection(&"127.0.0.1:14002".parse().unwrap()).await; + let before = manager.panel_streams().await; + assert!(before.iter().any(|row| row.preserved)); + assert_eq!(project(&manager).await, 3); + let after = manager.panel_streams().await; + assert_eq!(after.len(), 3); + assert!(after.iter().all(|row| row.provider.as_ref() == "account-b")); + assert_eq!(after.iter().filter(|row| row.preserved).count(), 1); +} + +#[tokio::test] +async fn handover_emits_one_update_per_changed_viewer() { + let manager = fixture().await; + let mut events = manager.events().get_event_channel(); + assert_eq!(project(&manager).await, 3); + assert_eq!(project(&manager).await, 0); + let mut received = Vec::new(); + while let Ok(event) = events.try_recv() { + match event { + EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(row)) => received.push(row), + other => panic!("projection must emit only full Updated events, got {other:?}"), + } + } + let mut rows = manager.panel_streams().await; + rows.sort_by_key(|row| row.uid); + received.sort_by_key(|row| row.uid); + assert_eq!(received, rows); +} + +#[tokio::test] +async fn provider_projection_preserves_stream_identity_and_accounting() { + let manager = fixture().await; + manager.release_connection(&"127.0.0.1:14002".parse().unwrap()).await; + let mut expected = manager.panel_streams().await; + let counts = manager.active_users_and_connections().await; + let resources = manager.playback_resource_counts().await; + let sessions = { + let connections = manager.connections.read().await; + connections + .by_key + .iter() + .map(|(user, data)| { + ( + user.clone(), + data.sessions + .iter() + .map(|session| { + ( + session.token.clone(), + session.provider.clone(), + session.ts, + session.started_at, + session.identity(), + session.lifecycle.clone(), + ) + }) + .collect::>(), + ) + }) + .collect::>() + }; + project(&manager).await; + for row in &mut expected { + row.provider = "account-b".intern(); + } + let mut actual = manager.panel_streams().await; + expected.sort_by_key(|row| row.uid); + actual.sort_by_key(|row| row.uid); + assert_eq!(actual, expected); + assert_eq!(manager.active_users_and_connections().await, counts); + assert_eq!(manager.playback_resource_counts().await, resources); + let connections = manager.connections.read().await; + let after = connections + .by_key + .iter() + .map(|(user, data)| { + ( + user.clone(), + data.sessions + .iter() + .map(|session| { + ( + session.token.clone(), + session.provider.clone(), + session.ts, + session.started_at, + session.identity(), + session.lifecycle.clone(), + ) + }) + .collect::>(), + ) + }) + .collect::>(); + assert_eq!(after, sessions); +} + +#[tokio::test] +async fn handover_only_updates_rows_of_the_matching_shared_session() { + let manager = fixture().await; + viewer(&manager, 4, "alice", Some(43)).await; + viewer(&manager, 5, "bob", None).await; + manager.terminate_session("alice", "session-3").await; + let before = manager.panel_streams().await; + assert_eq!(project(&manager).await, 2); + let mut expected = before; + for row in &mut expected { + if row.channel.shared_stream_id == Some(42) { + row.provider = "account-b".intern(); + } + } + let mut actual = manager.panel_streams().await; + expected.sort_by_key(|row| row.uid); + actual.sort_by_key(|row| row.uid); + assert_eq!(actual, expected); + assert!(actual.iter().all(|row| row.session_token.as_deref() != Some("session-3"))); +} + +#[derive(Clone, Copy)] +enum ViewerRelease { + Connection, + Stream, +} + +#[derive(Clone, Copy)] +enum ConcurrentViewerChange { + Handover, + NewerHandover, + Removal, + Reactivation, +} + +async fn reactivate_viewer(manager: &ActiveUserManager, row: &StreamInfo) { + let fingerprint = Fingerprint::new("viewer-2".into(), row.client_ip.clone(), row.addr); + manager + .update_connection(ActiveUserConnectionParams { + uid: 22, + meter_uid: row.meter_uid, + username: &row.username, + max_connections: 10, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: row.provider.clone(), + stream_channel: &row.channel, + user_agent: Cow::Borrowed(&row.user_agent), + session_token: row.session_token.as_deref(), + }) + .await + .expect("viewer reactivates the same row"); +} + +async fn assert_delayed_release_publishes_current_viewer(release_kind: ViewerRelease, change: ConcurrentViewerChange) { + let manager = fixture().await; + if matches!(change, ConcurrentViewerChange::NewerHandover) { + assert_eq!(project(&manager).await, 3); + } + let addr = "127.0.0.1:14002".parse().unwrap(); + let mut events = manager.events().get_event_channel(); + let expiry = manager.adaptive_expiry_index.lock().await; + let mut release = Box::pin(async { + match release_kind { + ViewerRelease::Connection => { + manager.release_connection(&addr).await; + } + ViewerRelease::Stream => { + assert!(matches!( + manager.release_stream_request_by_uid(&addr, 2).await, + super::StreamRequestDetach::Preserved(_) + )); + } + } + }); + assert!(futures::poll!(release.as_mut()).is_pending()); + assert!(manager.panel_streams().await.iter().any(|row| row.uid == 2 && row.preserved)); + let provider = match change { + ConcurrentViewerChange::NewerHandover => "account-c", + ConcurrentViewerChange::Handover | ConcurrentViewerChange::Removal | ConcurrentViewerChange::Reactivation => { + "account-b" + } + }; + assert_eq!(manager.lock_shared_stream_provider_projection(42).await.update_provider(&provider.intern()), 3); + match change { + ConcurrentViewerChange::Handover | ConcurrentViewerChange::NewerHandover => {} + ConcurrentViewerChange::Removal => { + assert!(manager.terminate_session("bob", "session-2").await); + // Any further Updated frame would resurrect the row after its removal event. + events = manager.events().get_event_channel(); + } + ConcurrentViewerChange::Reactivation => { + let row = manager.panel_streams().await.into_iter().find(|row| row.uid == 2).unwrap(); + reactivate_viewer(&manager, &row).await; + } + } + let expected = manager.panel_streams().await.into_iter().find(|row| row.uid == 2); + drop(expiry); + release.await; + let mut delivered = Vec::new(); + while let Ok(event) = events.try_recv() { + if let EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(row)) = event { + if row.uid == 2 { + delivered.push(row); + } + } + } + if let Some(expected) = expected { + assert_eq!( + delivered.last(), + Some(&expected), + "last delivered update must match the complete authoritative row" + ); + assert!(delivered.iter().all(|row| row.provider.as_ref() != "account-a"), "no old account after handover"); + } else { + assert_eq!(delivered, [], "a delayed release must not republish a removed viewer"); + } + assert_eq!(manager.panel_streams().await.iter().filter(|row| row.uid != 2).count(), 2); +} + +#[tokio::test] +async fn delayed_connection_release_cannot_restore_previous_provider() { + assert_delayed_release_publishes_current_viewer(ViewerRelease::Connection, ConcurrentViewerChange::Handover).await; +} + +#[tokio::test] +async fn delayed_stream_release_cannot_restore_previous_provider() { + assert_delayed_release_publishes_current_viewer(ViewerRelease::Stream, ConcurrentViewerChange::Handover).await; +} + +#[tokio::test] +async fn delayed_release_cannot_overwrite_newer_handover() { + for release in [ViewerRelease::Connection, ViewerRelease::Stream] { + assert_delayed_release_publishes_current_viewer(release, ConcurrentViewerChange::NewerHandover).await; + } +} + +#[tokio::test] +async fn delayed_release_does_not_republish_removed_viewer() { + for release in [ViewerRelease::Connection, ViewerRelease::Stream] { + assert_delayed_release_publishes_current_viewer(release, ConcurrentViewerChange::Removal).await; + } +} + +#[tokio::test] +async fn delayed_release_publishes_current_reactivated_viewer() { + for release in [ViewerRelease::Connection, ViewerRelease::Stream] { + assert_delayed_release_publishes_current_viewer(release, ConcurrentViewerChange::Reactivation).await; + } +} diff --git a/backend/session/src/active_user_manager/tests/request_claims.rs b/backend/session/src/active_user_manager/tests/request_claims.rs index 516c079e0..8450fb118 100644 --- a/backend/session/src/active_user_manager/tests/request_claims.rs +++ b/backend/session/src/active_user_manager/tests/request_claims.rs @@ -48,7 +48,7 @@ async fn affine_request_claims_keep_playback_until_oldest_body_finishes() { let removed = manager.release_stream_by_uid(&addr, 41).await; assert!(removed.as_ref().is_some_and(|stream| stream.uid == 41)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -89,7 +89,7 @@ async fn affine_request_claims_keep_playback_until_newest_body_finishes() { let removed = manager.release_stream_by_uid(&addr, 46).await; assert!(removed.as_ref().is_some_and(|stream| stream.uid == 45)); assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } fn shared_hls_channel(virtual_id: u32, shared_stream_id: u64) -> shared::model::StreamChannel { diff --git a/backend/session/src/active_user_manager/tests/socket_activity.rs b/backend/session/src/active_user_manager/tests/socket_activity.rs index 0af494fd2..52a61bc4c 100644 --- a/backend/session/src/active_user_manager/tests/socket_activity.rs +++ b/backend/session/src/active_user_manager/tests/socket_activity.rs @@ -284,13 +284,13 @@ async fn kicked_release_removes_preserved_adaptive_stream_without_socket_registr let released = manager.release_connection(&addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(released.removed_streams.len(), 0); + assert_eq!(manager.active_streams().await.len(), 0); let kicked = manager.release_connection_as_kicked(&addr).await; assert!(kicked.addr_removed); assert_eq!(kicked.removed_streams.len(), 1); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -564,7 +564,7 @@ async fn release_stream_by_uid_finds_original_user_after_shared_addr_owner_chang assert!(manager.release_stream_by_uid(&addr, 44).await.is_some()); assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -619,7 +619,7 @@ async fn release_connection_cleans_every_user_stream_for_reused_addr_only() { manager.release_connection(&unrelated_addr).await; assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -1188,7 +1188,7 @@ async fn vod_session_survives_overlapping_and_seek_sockets() { assert!(manager.release_stream(&range_addr).await.is_none()); let released = manager.release_connection(&range_addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); + assert_eq!(released.removed_streams.len(), 0); { let connections = manager.connections.read().await; @@ -1217,7 +1217,7 @@ async fn vod_session_survives_overlapping_and_seek_sockets() { assert!(manager.release_stream(&seek_addr).await.is_none()); let released = manager.release_connection(&seek_addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); + assert_eq!(released.removed_streams.len(), 0); let connections = manager.connections.read().await; let connection_data = connections.by_key.get("user1").expect("user connection data"); diff --git a/backend/session/src/connection_manager/cleanup.rs b/backend/session/src/connection_manager/cleanup.rs index 1deb00438..04c6d19a7 100644 --- a/backend/session/src/connection_manager/cleanup.rs +++ b/backend/session/src/connection_manager/cleanup.rs @@ -249,7 +249,7 @@ pub(super) async fn handle_update_detail_and_release_provider( ) .await; } - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + deps.user_manager.emit_current_stream_update(&stream_info).await; } if let Some(handle) = handle { deps.provider_manager.release_handle(&handle); diff --git a/backend/session/src/connection_manager/registration.rs b/backend/session/src/connection_manager/registration.rs index 2a912d907..c8c8fea46 100644 --- a/backend/session/src/connection_manager/registration.rs +++ b/backend/session/src/connection_manager/registration.rs @@ -6,7 +6,7 @@ use super::{ use crate::{uses_direct_body_idle_timeout, ActiveUserConnectionParams}; use log::warn; use shared::{ - model::{ActiveUserConnectionChange, CustomVideoStreamType, EventMessage, StreamInfo}, + model::{CustomVideoStreamType, StreamInfo}, utils::sanitize_sensitive_info, }; use std::{net::SocketAddr, sync::Arc}; @@ -170,8 +170,7 @@ impl ConnectionManager { ); self.touch_direct_body_activity(&fingerprint.addr); } - self.event_manager - .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); + self.user_manager.emit_current_stream_update(&stream_info).await; RegisteredPlaybackRequest::with_cleanup(uid, Some(stream_info), Some(cleanup)) } else { // Registration failed: nothing was inserted, so the cleanup must not fire. @@ -187,13 +186,13 @@ impl ConnectionManager { pub async fn update_stream_detail(&self, addr: &SocketAddr, video_type: CustomVideoStreamType) { if let Some(stream_info) = self.user_manager.update_stream_detail(addr, video_type).await { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.user_manager.emit_current_stream_update(&stream_info).await; } } pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) { if let Some(stream_info) = self.user_manager.update_stream_detail_by_uid(uid, video_type).await { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.user_manager.emit_current_stream_update(&stream_info).await; } } } diff --git a/backend/session/src/connection_manager/tests/lifecycle.rs b/backend/session/src/connection_manager/tests/lifecycle.rs index 90c18955f..03668eaa4 100644 --- a/backend/session/src/connection_manager/tests/lifecycle.rs +++ b/backend/session/src/connection_manager/tests/lifecycle.rs @@ -531,7 +531,7 @@ async fn shutdown_releases_active_streams() { }) .await; assert!(stream_info.is_some(), "registration must succeed"); - assert!(!manager.user_manager.active_streams().await.is_empty()); + assert_ne!(manager.user_manager.active_streams().await.len(), 0); manager.shutdown().await; assert!(manager.user_manager.active_streams().await.is_empty(), "shutdown must release active streams"); diff --git a/backend/session/src/event_manager.rs b/backend/session/src/event_manager.rs index d07b45da9..e5192a580 100644 --- a/backend/session/src/event_manager.rs +++ b/backend/session/src/event_manager.rs @@ -722,7 +722,7 @@ mod snapshot_and_ring_tests { manager.send_event(EventMessage::ConfigChange(ConfigType::Config)); manager.send_event(EventMessage::RecordingChanged); - assert!(manager.snapshot().is_empty()); + assert_eq!(manager.snapshot().len(), 0); } #[test] diff --git a/backend/session/src/event_manager/playlist_update.rs b/backend/session/src/event_manager/playlist_update.rs index a293d5ee3..d42647ccf 100644 --- a/backend/session/src/event_manager/playlist_update.rs +++ b/backend/session/src/event_manager/playlist_update.rs @@ -114,7 +114,7 @@ mod tests { manager.send_event(completed("r1", 2)); assert_eq!(manager.playlist_update_snapshot(), vec![own]); manager.send_event(completed("r1", 1)); - assert!(manager.playlist_update_snapshot().is_empty()); + assert_eq!(manager.playlist_update_snapshot().len(), 0); manager.send_event(EventMessage::PlaylistUpdateProgress(progress("r1", 1, 7))); assert!(manager.playlist_update_snapshot().is_empty(), "late progress cannot resurrect a finished input/run"); let next = progress("r2", 2, 7); @@ -184,7 +184,7 @@ mod tests { for event in [PlaylistUpdateProgressEvent::global("target", "working"), no_order, no_run] { manager.send_event(EventMessage::PlaylistUpdateProgress(event)); } - assert!(manager.playlist_update_snapshot().is_empty()); + assert_eq!(manager.playlist_update_snapshot().len(), 0); manager.send_event(EventMessage::PlaylistUpdateProgress(valid.clone())); manager.send_event(EventMessage::PlaylistUpdateProgress(progress("conflicting", 1, 7))); assert_eq!(manager.playlist_update_snapshot(), vec![valid]); diff --git a/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs b/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs index fd11d8588..f0a48acda 100644 --- a/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs +++ b/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs @@ -221,7 +221,7 @@ async fn shared_origin_abort_before_user_registration_releases_physical_and_logi .await?; assert_eq!(providers.get_provider_connections_count(), 0); assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); - assert!(users.active_streams().await.is_empty()); + assert_eq!(users.active_streams().await.len(), 0); Ok(()) } @@ -414,7 +414,7 @@ async fn shutdown_with_unpolled_shared_body_leaves_no_requests_or_subscribers() connections.shutdown().await; assert_eq!(providers.get_provider_connections_count(), 0); - assert!(users.active_streams().await.is_empty()); + assert_eq!(users.active_streams().await.len(), 0); assert!(manager.get_shared_state(url).await.is_none()); drop(unpolled_body); diff --git a/docs/src/configuration/hls-cache-state-machine.md b/docs/src/configuration/hls-cache-state-machine.md index ef27b40dc..ae0b1f55f 100644 --- a/docs/src/configuration/hls-cache-state-machine.md +++ b/docs/src/configuration/hls-cache-state-machine.md @@ -304,7 +304,7 @@ the old location. | Target duration fallback | 15 seconds | Used before an origin target duration is known. | | Publication lateness | `1.5 * target_duration` | Observation signal only; never a cutover or terminal deadline. | | Recovery trigger | Lease READY reserve minus full acceptance ETA and transition margin | Starts recovery while the configured burst can still complete. | -| Terminal cutover | Lease-specific safe commit deadline | Prevents terminal publication after the lease's usable reserve is exhausted. | +| Terminal cutover | Lease-specific safe splice deadline | Bounds a live-tail splice; a late finite Custom Response uses a separate bounded commit budget. | | HLS cache GC interval | 30 seconds | Periodic cleanup cadence for cache objects and stale runtime state. | | Temporary file retention | 30 seconds | Retention window for temporary HLS cache files. | | Failed segment retention | 10 seconds | Short retention for failed segment state. | diff --git a/docs/src/configuration/reverse-proxy.md b/docs/src/configuration/reverse-proxy.md index ed9e3a64a..bd6953dde 100644 --- a/docs/src/configuration/reverse-proxy.md +++ b/docs/src/configuration/reverse-proxy.md @@ -290,6 +290,12 @@ For the configuration reference, see [Shared HLS Configuration](./shared-hls-con For the shared session, access lease, and transient delivery state machines, see [HLS Cache State Machines](./hls-cache-state-machine.md). +The session and global byte limits cover committed objects, immutable revisions, and outstanding write reservations. +Staging and postprocessing grow their budgets before writing additional bytes. Committing an object consumes its own +reservation; capacity released by a completed or aborted write wakes deferred downloads. Both targeted reclamation +and periodic GC retire obsolete, unpinned revisions. See +[capacity diagnostics](./shared-hls-troubleshooting.md#symptom-downloads-wait-for-local-cache-capacity). + ```yaml reverse_proxy: hls_cache: @@ -316,6 +322,15 @@ reverse_proxy: max_parallel_jobs: 1 ``` +Short rolling origin playlists (one or two segments) are permitted only after the session evaluates its complete +configured manifest recovery burst. A suitable window with at least three segments takes priority. The resulting +permission belongs to the session's validated origin binding and epoch, and survives subsequent short manifests +and new access leases. Freshness refreshes share the existing singleflight; they do not themselves renew the burst. +Failed qualification also shares its retry receipt and backoff. A genuine origin revalidation requires a new +qualification. One warning per short-window qualification cites RFC 8216 section 6.2.2; larger available cache +windows, media readiness, lease validity, and resource retention rules still apply. See +[Shared HLS configuration](shared-hls-configuration.md#unstable-manifests). + ### HLS Cache Parameter Details | Parameter | Type | Default | Technical Impact | @@ -351,7 +366,20 @@ Important boundaries: * `reverse_proxy.hls_cache` only prepares the global cache engine. A target must also set `options.share_live_streams.hls: true` in `source.yml` before generated HLS live entries use the shared HLS path. -* HLS cache retry behavior is fixed internally and is not user-configurable. +* HLS cache retry behavior is fixed internally and is not user-configurable. Shared HLS origin-manifest retries cap an + upstream `Retry-After` delay at half the last known target duration. When no target duration is available, the + 15-second fallback limits the delay to 7.5 seconds. +* Playback reserve uses a stable session media clock and completed segment deliveries. Removing old cache entries + does not reset the playback position; missing segments, MAPs and keys still limit the usable reserve. +* A retryable manifest failure marks the origin as degraded on the first failed attempt. A bounded background owner + reevaluates the remaining reserve and schedules the terminal transition independently of origin retries and client + playlist polling. New origin progress invalidates the pending transition. +* A terminal video splice must commit before its safe splice deadline. After that deadline, a valid published lease + receives an HTTP `200` finite `channel_unavailable` Custom Response on the same manifest URL, containing only prepared + custom segments. This fallback needs no old cache segments and has a separate bounded commit budget that playlist + polls cannot renew. Successful authorized custom-segment requests renew the lease and session activity independently + of manifest polling after `ENDLIST`. Disabled, missing or invalid custom media can still produce an error response; + concrete preparation failures such as `TargetDurationExceeded` retain their diagnostic reason. * `reverse_proxy.rewrite_secret` must stay stable. It is used for future HLS `proxy_session_id` values and transient resource IDs. * `session_idle_timeout` controls HLS cache access-lease validity and idle cleanup. It is independent from `reverse_proxy.stream.hls_session_ttl_secs`, which belongs to the non-cache HLS request continuity path. @@ -590,6 +618,24 @@ reverse_proxy: ## Additional Information +### Shared HLS account replacement + +A shared HLS session checks its bound account when it needs origin work. Removed, expired, disabled and runtime-blocked +accounts are replaced using available accounts from the same input pool. Manifest refreshes, segment and MAP cache misses, +key/transient fetches and existing background workers use the same preparation path. Cached media needs no new allocation. +There is no additional monitoring service or configuration setting. + +Replacement retains the shared session identity, valid access leases, published timeline and cached media. Tuliprox drops +old provider cookies and recovery targets, resolves the entry URL with the replacement account, and qualifies its manifest +against the existing timeline and content/host checks before scheduling new media. Catchup retains its archive identity. +Account replacement assigns new resource IDs even when the concrete key URI is unchanged. Cached AES keys for previously +published media retain their old IDs, so the replacement manifest cannot reuse a key cached for the previous account. + +If no suitable account is available, requests receive the existing temporary response with `Retry-After`; background work +uses its existing retry budgets. Failed replacement attempts are separated by at least two seconds. Full capacity, foreign +reservations, timeouts and isolated HTTP errors do not invalidate an otherwise healthy account. Replacement never borrows +an account from another source or guarantees uninterrupted playback. + ### Session TTLs for HLS (`.m3u8`) & Catchup HLS streams do not consist of an endless TCP pipe. Instead, the player downloads small `.ts` segments every few seconds (e.g., `seg1.ts`, `seg2.ts`). diff --git a/docs/src/configuration/shared-hls-configuration.md b/docs/src/configuration/shared-hls-configuration.md index 869539f5c..d963f259d 100644 --- a/docs/src/configuration/shared-hls-configuration.md +++ b/docs/src/configuration/shared-hls-configuration.md @@ -197,6 +197,21 @@ reverse_proxy: Use recovery burst levels gradually. Start with `friendly`. Move to stronger levels only when logs show manifest refresh instability and the provider can tolerate additional requests. +Shared HLS evaluates a short live origin window (one or two segments) with the complete configured burst before +allowing it. A suitable candidate with at least three segments takes priority. If only suitable short windows +remain, the session records permission to publish them and emits one warning referencing RFC 8216 section 6.2.2. +The `off` level evaluates its single candidate; no additional burst level is required. + +All access leases share the qualification and refresh work, including failure backoff. New leases, ordinary +sequence movement, and later changes such as one → three → one segments do not consume or renew the permission. +A request that needs a fresher manifest uses the normal shared refresh. A changed origin binding or epoch, +acceptance conflict, or lifecycle state requiring origin revalidation starts a new shared qualification. + +The proxy continues to use larger available cache windows. Strip keeps already short windows intact, and +startup admission and reserve accounting use their actual media duration. Missing segments, maps or keys, +replayed media, empty playlists, and invalid leases remain subject to the existing checks. This exception cannot +create media progress when the origin stops publishing. + ### Problematic MPEG-TS segments ```yaml @@ -290,8 +305,9 @@ or silently change the bytes of an already published URI. Revision and repair pe disk cache limits. Conservative behavior remains the default; existing sessions keep their startup policy after reload. Restart after changing to `conservative` when an immediate rollback of running sessions is required. -Existing provider and fetch concurrency limits remain in effect. The initial playlist needs at least three -publishable segments; `optimistic` counts successors the origin has accepted. With fewer than three concurrent +Existing provider and fetch concurrency limits remain in effect. The initial playlist normally needs at least three +publishable segments; a session qualified for short origin windows can publish one or two. `optimistic` counts +successors the origin has accepted. With fewer than three concurrent fills, a cold session can still wait for an earlier origin EOF before that window is available. Raising the existing per-session concurrency limit to three is an operator choice and must fit the provider's connection allowance; startup mode never raises it automatically. diff --git a/docs/src/configuration/shared-hls-runtime-flow.md b/docs/src/configuration/shared-hls-runtime-flow.md index 69d67b2c5..f954a1308 100644 --- a/docs/src/configuration/shared-hls-runtime-flow.md +++ b/docs/src/configuration/shared-hls-runtime-flow.md @@ -211,13 +211,13 @@ runs the parser or writer. The prepared-bundle key binds: One ready bundle contains twelve small immutable terminal blocks. For block index `x`, timestamp progression is: ```text -offset_90khz(x) = x * target_duration_ms * 90 +offset_90khz(x) = x * asset_duration_ticks_90khz offset_pcr(x) = offset_90khz(x) * 300 ``` -The target duration is intentionally the timestamp stride; the measured terminal asset duration is the `#EXTINF` value. -When that asset is shorter than the target duration, the resulting bounded timestamp gap is intentional. There is no old -`index * asset_duration_ticks` terminal calculation. +The measured terminal asset duration determines the timestamp stride and the `#EXTINF` value. The target duration is +retained as the playlist limit; a shorter asset does not introduce a timestamp gap between custom segments. Both the +live-tail splice and the late Custom Response use this same prepared bundle. ## Warm terminal-tail response @@ -234,7 +234,27 @@ the optional key reset, `#EXT-X-DISCONTINUITY`, twelve finite terminal segment r /hls/shared/live///terminal//.ts ``` -Only that lease- and generation-bound route serves prepared terminal bytes and finite full/range responses. Existing +If the safe splice deadline has already elapsed, a valid lease with a previously published manifest instead commits a +finite Custom Response without a live suffix. Its canonical manifest returns HTTP `200` with no redirect, twelve +prepared `channel_unavailable` segments under `/cvs/hls///.ts`, a clear-key +reset, a discontinuity and `#EXT-X-ENDLIST`. The target duration is retained; media sequence starts after the highest +segment number previously published to that lease, and discontinuity sequence does not decrease. No live cache base, +origin request or timestamp rewrite is needed to serve these custom segments. Playback may stall before this late +warning becomes visible: the fallback does not promise a seamless transition after the deadline. + +The late commit has one bounded retry budget independent of the expired splice deadline. Repeated playlist requests +cannot extend it. Lease incarnation, publication/cursor generation, origin recovery and configured asset revision +are revalidated at commit. The frozen manifest and media remain unchanged on replay and asset reload; access still +requires the valid lease and existing user authorization. Successful authorized GET, HEAD and range requests for the +late custom segments activate or renew that exact lease and refresh session activity using the configured +`session_idle_timeout`; playback does not depend on further manifest polls after `#EXT-X-ENDLIST`. Invalid routes, +unsatisfiable ranges and denied authorization do not renew the lease, and an expired lease cannot be reactivated. +Disabled, missing or incompatible custom media can retain an error response. Preparation failures retain their typed +compatibility reason, including `TargetDurationExceeded` with the measured asset and playlist target durations. +The original compatible live-tail splice continues to apply before its deadline. + +Only the lease- and generation-bound terminal route serves the live-tail splice's prepared terminal bytes and finite +full/range responses. Existing normal segment, MAP, and `/r` URLs retain their original meaning and never redirect to a manifest or substitute slate bytes. Terminal playback mode is sticky for the lease; later shared-session recovery can serve other or new leases but cannot reactivate the terminal lease. @@ -244,6 +264,36 @@ cannot reactivate the terminal lease. The shared session can hold an origin account binding. That binding represents the provider account or provider connection Tuliprox uses for origin work. +Before necessary origin I/O, a session advances overlap promotion, detachment and reclaim, then re-reads the current input +configuration and binding. Account replacement is serialized per session. Missing, expired and blocked accounts share one replacement path; +blocked includes disabled accounts and runtime +health observations before persistence. Allocation stays inside the original input and follows existing capacity and +priority rules. An absent input cannot migrate to another source. A failed attempt retains its two-second backoff even +when the binding has become detached. + +The HLS worker boundary uses a typed future-returning prepare callback. Its app adapter holds a weak application-state +reference. Manifest, segment, MAP, key/transient and existing background work prepare before capturing a fetch generation +or snapshot. Only authorized live work may prepare; terminal/expired leases and cache hits do not allocate. +Preparation returns the resolved entry together with its account identity, work generation and provider headers. Refresh +initialization carries that result intact; a changed binding or generation requires preparation again or rejects the start. +An unused allocation for the previous entry is released before preparing its replacement. + +An account handoff advances account/work generations, invalidates waiting origin work, old cookies, recovery targets and +unfetched resource bindings, and keeps session identity, access leases, archive identity, timeline and cached objects. +Media work first requests manifest qualification through the existing refresh flow. The first request resolves the new +entry, including `provider://` entries; any subsequent recovery is bound exclusively to that new concrete HTTP(S) target. +Continuity, content-anchor and cross-host checks still apply. Old in-flight results and tagged cleanup cannot publish over +or release a successor binding. After releasing an old worker permit, the existing scheduler resumes qualified successor +work, including when only one global fetch permit exists. Allocation handles own their release responsibility across +asynchronous preparation. +Each refresh also owns a separate singleflight token. An invalidated refresh releases its own flag even after a failed +replacement, while a late completion cannot clear a successor token, including one started in the same millisecond. + +Account handoff gives newly published transient resources and normal AES key references a separate resource-ID namespace. +Identical origin key URLs therefore cannot inherit cached keys from the previous account. Old published references keep +their original mappings and cached bytes; existing resource TTLs and metadata limits still govern retention. Rolling EVENT +rewrites restart at handoff and preserve the new namespace when appending subsequent manifest lines. + Origin account protection is based on successful media access: | Protection state | Meaning | diff --git a/docs/src/configuration/shared-hls-troubleshooting.md b/docs/src/configuration/shared-hls-troubleshooting.md index ea945fabd..6b928aab9 100644 --- a/docs/src/configuration/shared-hls-troubleshooting.md +++ b/docs/src/configuration/shared-hls-troubleshooting.md @@ -90,7 +90,11 @@ Fix: For the warm Shared HLS path, this is a lease-local terminal decision rather than a failure-counter threshold. The canonical manifest stays on the same URL and returns an HTTP `200` finite playlist when a compatible terminal tail is -ready. An incompatible MAP/key/container/track state fails closed with `503`; it never creates an unsafe TS splice. +ready. Before the splice deadline, an incompatible MAP/key/container/track state can fail closed with `503`. +After the deadline, a valid published lease can receive a finite Custom Response containing only prepared +`channel_unavailable` media, even when its live cache suffix is gone. The playlist uses `/cvs/hls/` URLs and does not +attempt a late TS splice. The player may stall before displaying it. Disabled, missing or invalid custom media, +expired authorization, and exhausted commit capacity can still prevent a usable response. Possible causes: @@ -187,6 +191,38 @@ Fix: - Lower `cache_duration` if unprotected old objects stay too long. - Put `cache_path` on a disk with enough free space and predictable cleanup behavior. +## Symptom: downloads wait for local cache capacity + +`LocalCacheCapacity` means the decoded body cannot fit the current session or global budget. The budget includes +committed segments, MAPs, transient objects, immutable revisions, and the remaining reservations of active writes. +Startup reserves the existing peak allowance (three body copies for processed revisions, five for raw revisions). +Committing a file transfers its bytes from that write's reservation into committed usage; it does not add the same +bytes twice. Unknown or growing bodies reserve additional space before writing it. +Replacing an object with a smaller file releases the old bytes only after the atomic rename succeeds. + +Use these debug messages to distinguish the causes: + +| Log signal | Meaning | +| :--- | :--- | +| `HLS capacity inventory` | Per-session `revision_bytes` and `remaining_reserved_bytes`, plus revision deletion outcomes. | +| `HLS capacity FIFO` | The oldest segment, its status, and its actual blocker: protection, pending work, or no candidates. | +| `outcome=lock-conflict protected_working_set_known=false` | Selection could not acquire the session lock. A zero protection estimate is unknown, not evidence that every byte is reclaimable. | +| `HLS revision GC` | Bounded deletion attempts: `planned`, `succeeded`, `failed`, and actual `reclaimed_bytes`. | +| `HLS revision delete deferred` | Filesystem error; the revision stays queued for a later attempt and contributes no reclaimed bytes. | + +Both periodic GC and targeted admission release obsolete references and retire unpinned revisions. A protected FIFO +head still stops segment eviction. An inactive `CapacityDeferred` head can be retired once its origin binding and +all protections are gone. Open readers and retained lease publications keep their revision bytes pinned. + +A capacity waiter resumes when reservation, accounting, or playback protection changes, then checks the complete +peak of a fresh attempt before downloading again. Releasing too little capacity does not start another origin fetch. +It does not need a periodic GC tick or traffic from another session. Published revision bytes remain immutable across +repeated requests and ranges, even when the mutable normal cache object is replaced or rolled back. + +The in-memory `revision_disk_bytes` gauge is updated by successful file mutations and inventory reconciliation. +Revision deletion counters distinguish attempts, successes, failures, and bytes actually reclaimed. Inspect sustained +I/O failures or retained readers before changing limits. Do not copy credential-bearing origin URLs into diagnostics. + ## Symptom: cache path permission errors The Tuliprox process must be able to create directories, write temporary files, rename committed cache objects, read cached diff --git a/frontend/src/app/components/playlist/epg_view/mod.rs b/frontend/src/app/components/playlist/epg_view/mod.rs index 100535104..df100aed0 100644 --- a/frontend/src/app/components/playlist/epg_view/mod.rs +++ b/frontend/src/app/components/playlist/epg_view/mod.rs @@ -1342,6 +1342,6 @@ mod tests { #[test] fn pending_program_source_id_does_not_treat_numeric_epg_id_as_playlist_id() { let full = EpgGridModel::from_epg_tv(&EpgTv::new(vec![EpgChannel::new("123".intern())]), |_| 0); - assert!(pending_program_source_id(&full.channels[0]).is_empty()); + assert_eq!(pending_program_source_id(&full.channels[0]).len(), 0); } } diff --git a/frontend/src/app/components/playlist/input_update_card.rs b/frontend/src/app/components/playlist/input_update_card.rs index 518ec6d9f..7860a9a7c 100644 --- a/frontend/src/app/components/playlist/input_update_card.rs +++ b/frontend/src/app/components/playlist/input_update_card.rs @@ -1812,7 +1812,7 @@ mod tests { TargetSelectionState::All ); let cleared = reduce(individually_restored, InputUpdateCardAction::ToggleAll(targets.clone())); - assert!(cleared.controls.selected_target_ids.is_empty()); + assert_eq!(cleared.controls.selected_target_ids.len(), 0); assert_eq!( target_selection_state(&cleared.controls.selected_target_ids, &targets), TargetSelectionState::Empty @@ -2260,9 +2260,9 @@ mod tests { let replacement = card(7, &[4]); let targetless = card(7, &[]); - assert!(selected_current_target_ids(&replacement, &state.controls.selected_target_ids).is_empty()); + assert_eq!(selected_current_target_ids(&replacement, &state.controls.selected_target_ids).len(), 0); assert!(!can_start_update(&state, &replacement, true)); - assert!(selected_current_target_ids(&targetless, &state.controls.selected_target_ids).is_empty()); + assert_eq!(selected_current_target_ids(&targetless, &state.controls.selected_target_ids).len(), 0); assert!(!can_start_update(&state, &targetless, true)); } diff --git a/frontend/src/app/components/playlist/playlist_update_view.rs b/frontend/src/app/components/playlist/playlist_update_view.rs index 344be35b0..f9c219d3b 100644 --- a/frontend/src/app/components/playlist/playlist_update_view.rs +++ b/frontend/src/app/components/playlist/playlist_update_view.rs @@ -579,7 +579,7 @@ mod tests { assert_eq!(state.for_input(input_id).details(), [detail]); } assert_eq!(state.for_input(9).status(), InputUpdateCardStatus::Ready); - assert!(state.for_input(9).details().is_empty()); + assert_eq!(state.for_input(9).details().len(), 0); let library = LibraryScanProgressEvent { summary: LibraryScanSummary { status: LibraryScanSummaryStatus::Success, diff --git a/frontend/src/app/components/source_editor/output_xtream_form.rs b/frontend/src/app/components/source_editor/output_xtream_form.rs index d7c9a8fbb..bb9bfa4c3 100644 --- a/frontend/src/app/components/source_editor/output_xtream_form.rs +++ b/frontend/src/app/components/source_editor/output_xtream_form.rs @@ -833,7 +833,7 @@ mod tests { .expect("lists-only trakt config"); assert_eq!(result.lists, lists); - assert!(result.charts.is_empty()); + assert_eq!(result.charts.len(), 0); } #[test] @@ -859,7 +859,7 @@ mod tests { assert_eq!(result.catalog_selection, TraktCatalogSelection::Curated); assert!(!result.include_xtream_base_categories); - assert!(result.lists.is_empty()); + assert_eq!(result.lists.len(), 0); assert_eq!(result.charts, charts); } diff --git a/frontend/src/hooks/use_server_status.rs b/frontend/src/hooks/use_server_status.rs index 124074f8f..bc9e84e29 100644 --- a/frontend/src/hooks/use_server_status.rs +++ b/frontend/src/hooks/use_server_status.rs @@ -204,7 +204,6 @@ fn replace_server_status_snapshot( // Trust backend `panel_streams` as authoritative. Re-merging sticky FE ghosts resurrected // ended archive/HLS rows until a full page reload. dedupe_streams_by_identity(&mut server_status.active_user_streams); - prune_zapped_preserved_streams(&mut server_status.active_user_streams); merge_aux_streams(&mut server_status, download_streams); let server_status = Rc::new(server_status); *status_holder = Some(Rc::clone(&server_status)); @@ -255,14 +254,22 @@ fn apply_active_user_change(server_status: &mut StatusCheck, event: ActiveUserCo } return; } - if let Some(pos) = find_stream_update_index(&server_status.active_user_streams, &stream_info) { - server_status.active_user_streams[pos] = stream_info; - } else { - server_status.active_user_streams.push(stream_info); - } + let provider_only_update = + if let Some(pos) = find_stream_update_index(&server_status.active_user_streams, &stream_info) { + let mut previous = server_status.active_user_streams[pos].clone(); + previous.provider.clone_from(&stream_info.provider); + let provider_only_update = previous == stream_info; + server_status.active_user_streams[pos] = stream_info; + provider_only_update + } else { + server_status.active_user_streams.push(stream_info); + false + }; dedupe_streams_by_identity(&mut server_status.active_user_streams); // Channel zap: new active channel replaces preserved soft-kept previous channel immediately. - prune_zapped_preserved_streams(&mut server_status.active_user_streams); + if !provider_only_update { + prune_zapped_preserved_streams(&mut server_status.active_user_streams); + } } ActiveUserConnectionChange::Disconnected(addr) => { let mut retained = Vec::with_capacity(server_status.active_user_streams.len()); @@ -500,6 +507,58 @@ mod tests { stream } + #[test] + fn provider_update_replaces_existing_shared_hls_rows() { + let mut viewers = vec![ + test_shared_hls_stream(11, "127.0.0.1:11001", Some("viewer-a")), + test_shared_hls_stream(12, "127.0.0.1:11002", Some("viewer-b")), + test_shared_hls_stream(13, "127.0.0.1:11003", Some("viewer-c")), + ]; + viewers[1].preserved = true; + for row in &mut viewers { + row.channel.shared_stream_id = Some(42); + row.provider = "account-a".intern(); + row.started_at = 123; + row.ts = 456; + } + let mut status = shared::model::StatusCheck { active_user_streams: viewers.clone(), ..Default::default() }; + let viewer_count = viewers.len(); + for row in &mut viewers { + row.provider = "account-b".intern(); + apply_active_user_change(&mut status, ActiveUserConnectionChange::Updated(row.clone())); + assert_eq!( + status.active_user_streams.len(), + viewer_count, + "provider updates must never prune another viewer" + ); + } + assert_eq!(status.active_user_streams, viewers); + } + + #[test] + fn status_snapshot_replaces_stale_provider_names() { + let mut viewers = vec![ + test_shared_hls_stream(11, "127.0.0.1:11001", Some("viewer-a")), + test_shared_hls_stream(12, "127.0.0.1:11002", Some("viewer-b")), + ]; + viewers[1].preserved = true; + for row in &mut viewers { + row.provider = "account-a".intern(); + } + let mut holder = + Some(Rc::new(shared::model::StatusCheck { active_user_streams: viewers.clone(), ..Default::default() })); + for row in &mut viewers { + row.provider = "account-b".intern(); + } + let result = replace_server_status_snapshot( + &mut holder, + shared::model::StatusCheck { active_user_streams: viewers.clone(), ..Default::default() }, + &[], + ); + assert_eq!(result.active_user_streams, viewers); + assert_eq!(holder.unwrap().active_user_streams, viewers); + } + #[test] fn test_find_stream_update_index_prefers_adaptive_session_token_over_addr() { let existing = test_stream(1, "127.0.0.1:1234", Some("tok-hls"), PlaylistItemType::LiveHls); diff --git a/frontend/src/model/input_update_capabilities.rs b/frontend/src/model/input_update_capabilities.rs index 51e342003..a674742e0 100644 --- a/frontend/src/model/input_update_capabilities.rs +++ b/frontend/src/model/input_update_capabilities.rs @@ -79,7 +79,7 @@ mod tests { #[test] fn input_update_capabilities_rescan_is_not_a_provider_policy_and_uses_targets() { let capability = InputUpdateCapabilities::Rescan; - assert!(capability.policies().is_empty()); + assert_eq!(capability.policies().len(), 0); assert!(capability.rebuilds_targets()); assert_eq!(capability.selected_action(InputRefreshPolicy::NORMAL), InputUpdateAction::Rescan); for unavailable in [ diff --git a/frontend/src/model/input_update_card_model.rs b/frontend/src/model/input_update_card_model.rs index e6ca9b1da..e3ecbf70b 100644 --- a/frontend/src/model/input_update_card_model.rs +++ b/frontend/src/model/input_update_card_model.rs @@ -221,7 +221,7 @@ mod tests { let cards = build_input_update_card_models(&sources, &status); let card = cards.iter().find(|card| card.input_id == 40).expect("card without targets"); - assert!(card.targets.is_empty()); + assert_eq!(card.targets.len(), 0); } #[test] @@ -237,7 +237,7 @@ mod tests { assert_eq!(cards.len(), 1); assert_eq!(cards[0].input_id, 7); assert_eq!(cards[0].input_type, InputType::Xtream); - assert!(cards[0].targets.is_empty()); + assert_eq!(cards[0].targets.len(), 0); } #[test] diff --git a/frontend/src/model/pipeline_transparency_model.rs b/frontend/src/model/pipeline_transparency_model.rs index 33b19a458..b76d4964a 100644 --- a/frontend/src/model/pipeline_transparency_model.rs +++ b/frontend/src/model/pipeline_transparency_model.rs @@ -486,7 +486,7 @@ mod tests { assert_eq!(reloaded.overall_state, InputUpdateCardStatus::Ready); assert_eq!(reloaded.run_id, None); assert_eq!(reloaded.cache_source, None); - assert!(reloaded.cluster_results.is_empty()); + assert_eq!(reloaded.cluster_results.len(), 0); } } } @@ -601,7 +601,7 @@ mod tests { assert_eq!(view.cache_source, None); assert_eq!(view.refresh_policy, None); assert_eq!(view.library_scan_result, None); - assert!(view.progress_details.is_empty()); + assert_eq!(view.progress_details.len(), 0); assert_eq!(view.cluster_results.is_empty(), !(input_type.is_xtream() || input_type.is_stalker())); assert!(view.cluster_results.iter().all(|cluster| cluster.outcome.is_none() && cluster.quality.is_none() @@ -713,7 +713,7 @@ mod tests { let view = build_input_update_run_views(&sources, &cards, &reloaded, &persisted).remove(0); assert_eq!(view.input_state, InputUpdateCardStatus::Success); assert_eq!(view.run_id, None); - assert!(view.progress_details.is_empty()); + assert_eq!(view.progress_details.len(), 0); assert_eq!(view.refresh_policy, None); } } @@ -761,7 +761,7 @@ mod tests { let reload = project(&fresh); assert_eq!(reload.library_catalog, Some(catalog.clone())); assert_eq!(reload.library_scan_result, None); - assert!(reload.cluster_results.is_empty()); + assert_eq!(reload.cluster_results.len(), 0); assert_eq!(reload.last_update_at, Some(1234)); let state = fresh .reduce(accepted("r1")) @@ -788,7 +788,7 @@ mod tests { assert_eq!(queued.run_id, Some("r2".into())); assert_eq!(queued.overall_state, InputUpdateCardStatus::Queued); assert_eq!(queued.library_scan_result, None); - assert!(queued.progress_details.is_empty()); + assert_eq!(queued.progress_details.len(), 0); assert_eq!(project(&state.clone().reduce(accepted("r1"))), queued); let state = state.reduce(progress(PlaylistUpdateProgressEvent::for_run_input( "r2".into(), @@ -1532,7 +1532,7 @@ mod tests { assert_eq!(view.refresh_policy, None); assert_eq!(view.cache_source, None); assert_eq!(view.last_update_at, None); - assert!(view.progress_details.is_empty()); + assert_eq!(view.progress_details.len(), 0); assert!(view.cluster_results.iter().all(|cluster| cluster.threshold.is_none())); assert_eq!(view.target_results[0].target_id, 99); assert_eq!(view.target_results[0].pipeline, None); @@ -1701,7 +1701,7 @@ mod tests { assert_eq!(queued.run_id.as_ref().map(PlaylistUpdateRunId::as_ref), Some("force-run")); assert_eq!(queued.overall_state, InputUpdateCardStatus::Queued); assert_eq!(queued.refresh_policy, Some(InputRefreshPolicy::FORCE)); - assert!(queued.progress_details.is_empty()); + assert_eq!(queued.progress_details.len(), 0); assert_eq!(queued.target_results, partial.target_results); let live = queued.cluster_results[0]; assert_eq!(live.requested, None); diff --git a/frontend/src/model/playlist_update_card_status.rs b/frontend/src/model/playlist_update_card_status.rs index 3fd329825..556f0ff9a 100644 --- a/frontend/src/model/playlist_update_card_status.rs +++ b/frontend/src/model/playlist_update_card_status.rs @@ -990,7 +990,7 @@ mod tests { playlist_update_bulk_scope(&sources, &cards), PlaylistUpdateBulkScope { can_submit: true, visible_input_ids: vec![7] } ); - assert!(local_controls.selected_target_ids.is_empty()); + assert_eq!(local_controls.selected_target_ids.len(), 0); assert_eq!(local_controls.policy, InputRefreshPolicy::FORCE); } @@ -1315,7 +1315,7 @@ mod tests { assert_eq!(queued.status(), InputUpdateCardStatus::Queued); assert_eq!(queued.run_id().map(PlaylistUpdateRunId::as_ref), Some("r2")); assert_eq!(queued.refresh_policy(), Some(InputRefreshPolicy::FORCE)); - assert!(queued.details().is_empty()); + assert_eq!(queued.details().len(), 0); assert_eq!(queued.input_telemetry(), None); } @@ -1425,7 +1425,7 @@ mod tests { let runtime = state.for_input(7); assert_eq!(runtime.status(), InputUpdateCardStatus::Failed); - assert!(runtime.details().is_empty()); + assert_eq!(runtime.details().len(), 0); let terminal = state.terminal_by_input_id.get(&7).expect("terminal state"); assert_eq!(terminal.run_id, run_id("r2")); assert_eq!(terminal.execution_order, execution_order(2)); diff --git a/frontend/src/model/staged_completion_tests.rs b/frontend/src/model/staged_completion_tests.rs index 1d9bd9d2f..7f19496bc 100644 --- a/frontend/src/model/staged_completion_tests.rs +++ b/frontend/src/model/staged_completion_tests.rs @@ -88,7 +88,7 @@ fn playlist_update_status_pipeline_transparency_reload_restores_running_target_r let queued = build_input_update_run_views(&sources, &cards, &state, &status).remove(0); assert_eq!(queued.run_id, Some("follow-up".into())); assert_eq!(queued.overall_state, InputUpdateCardStatus::Queued); - assert!(queued.progress_details.is_empty()); + assert_eq!(queued.progress_details.len(), 0); assert!(state.for_input(8).input_telemetry().is_none()); } } @@ -191,7 +191,7 @@ fn playlist_update_status_pipeline_transparency_real_staged_trace_keeps_own_resu "persisted input result proves no target/run result" ); assert_eq!(reload.run_id, None); - assert!(reload.progress_details.is_empty()); + assert_eq!(reload.progress_details.len(), 0); } } diff --git a/frontend/src/services/playlist_service.rs b/frontend/src/services/playlist_service.rs index 036ea2776..4867a72e7 100644 --- a/frontend/src/services/playlist_service.rs +++ b/frontend/src/services/playlist_service.rs @@ -312,7 +312,7 @@ mod tests { assert_eq!(request.target_ids, Some(vec![20, 21])); assert_eq!(request.input_action, Some(action)); assert_eq!(request.input_refresh, None); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(build_playlist_update_bulk_request().input_action, None); } @@ -334,7 +334,7 @@ mod tests { for &policy in InputUpdateCapabilities::for_input(&input).policies() { let request = build_input_playlist_update_request(&[30, 20], InputRefreshOverride { input_id: input.id, policy }); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, Some(vec![30, 20])); assert_eq!(request.input_refresh, Some(InputRefreshOverride { input_id: 7, policy })); } @@ -406,7 +406,7 @@ mod tests { InputRefreshOverride { input_id: 7, policy: InputRefreshPolicy::FORCE }, ); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, Some(vec![30, 20])); assert_eq!( request.input_refresh, @@ -422,7 +422,7 @@ mod tests { InputRefreshOverride { input_id: 7, policy: InputRefreshPolicy::NORMAL }, ); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, Some(expected_ids)); } } @@ -440,7 +440,7 @@ mod tests { fn playlist_update_bulk_request_uses_normal_policy_and_complete_server_target_selection() { let request = build_playlist_update_bulk_request(); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, None); assert_eq!(request.input_refresh, None); } diff --git a/shared/src/model/auth/role.rs b/shared/src/model/auth/role.rs index 22f8ccad8..c8459eb68 100644 --- a/shared/src/model/auth/role.rs +++ b/shared/src/model/auth/role.rs @@ -120,7 +120,7 @@ mod tests { let set = RoleSet::new(); assert!(set.is_empty()); assert!(!set.contains(Role::Admin)); - assert!(set.names().is_empty()); + assert_eq!(set.names().len(), 0); } #[test] diff --git a/shared/src/model/config/curation.rs b/shared/src/model/config/curation.rs index dcada51b0..227b0b689 100644 --- a/shared/src/model/config/curation.rs +++ b/shared/src/model/config/curation.rs @@ -195,7 +195,7 @@ mod tests { dto.prepare(&xtream()).expect("missing token is not a config-load failure"); assert!(dto.trakt.is_none()); let tmdb = dto.tmdb.unwrap(); - assert!(tmdb.api.access_token.is_empty()); + assert_eq!(tmdb.api.access_token.len(), 0); assert!(tmdb.enabled); assert_eq!(tmdb.trending[0].kind, TmdbTrendingKind::Movie); assert_eq!(tmdb.trending[0].time_window, TmdbTrendingTimeWindow::Week); diff --git a/shared/src/model/config/epg/tests.rs b/shared/src/model/config/epg/tests.rs index 010632123..15abe1a9b 100644 --- a/shared/src/model/config/epg/tests.rs +++ b/shared/src/model/config/epg/tests.rs @@ -52,7 +52,7 @@ fn include_computed_false_skips_resolution() { let mut cfg = EpgConfigDto { sources: Some(vec![xmltv_source("provider://myprovider/xmltv.php")]), ..Default::default() }; cfg.prepare(|| Err("no auto".to_owned()), false).expect("prepare with include_computed=false should succeed"); - assert!(cfg.t_sources.is_empty()); + assert_eq!(cfg.t_sources.len(), 0); } #[test] diff --git a/shared/src/model/config/target/tests.rs b/shared/src/model/config/target/tests.rs index 5b997d299..5cf38f5ad 100644 --- a/shared/src/model/config/target/tests.rs +++ b/shared/src/model/config/target/tests.rs @@ -38,7 +38,7 @@ fn shipped_tmdb_example_is_safe_by_default_and_prepares_when_explicitly_enabled( .expect("shipped TMDB example"); assert!(!target.enabled); assert!(!target.curation.as_ref().unwrap().enabled); - assert!(target.curation.as_ref().unwrap().tmdb.as_ref().unwrap().api.access_token.is_empty()); + assert_eq!(target.curation.as_ref().unwrap().tmdb.as_ref().unwrap().api.access_token.len(), 0); target.prepare(1, None, None).unwrap(); target.enabled = true; target.curation.as_mut().unwrap().enabled = true; diff --git a/shared/src/model/config/trakt.rs b/shared/src/model/config/trakt.rs index 33a47d737..288247d93 100644 --- a/shared/src/model/config/trakt.rs +++ b/shared/src/model/config/trakt.rs @@ -311,7 +311,7 @@ mod tests { assert_eq!(config.catalog_selection, TraktCatalogSelection::Full); assert!(config.include_xtream_base_categories); - assert!(config.lists.is_empty()); + assert_eq!(config.lists.len(), 0); assert_eq!(config.charts.len(), 1); assert_eq!(config.charts[0].kind, TraktChartKind::Movies); assert_eq!(config.charts[0].kind.content_type(), TraktContentType::Vod); diff --git a/shared/src/model/playlist_update_status.rs b/shared/src/model/playlist_update_status.rs index a1c5583a5..b71620b2c 100644 --- a/shared/src/model/playlist_update_status.rs +++ b/shared/src/model/playlist_update_status.rs @@ -188,8 +188,8 @@ mod tests { let decoded = serde_json::from_str::(r#"{"inputs":[{"input_id":7,"last_update":1725000000}]}"#)?; - assert!(decoded.inputs[0].clusters.is_empty()); - assert!(decoded.active_updates.is_empty()); + assert_eq!(decoded.inputs[0].clusters.len(), 0); + assert_eq!(decoded.active_updates.len(), 0); Ok(()) } diff --git a/shared/src/model/stalker.rs b/shared/src/model/stalker.rs index 42d9ebddf..6c19e0328 100644 --- a/shared/src/model/stalker.rs +++ b/shared/src/model/stalker.rs @@ -495,7 +495,7 @@ mod tests { assert!(!descriptor.is_empty()); descriptor.clean(); // clean() drops empty candidates, so the descriptor is now fully empty. - assert!(descriptor.candidates.is_empty()); + assert_eq!(descriptor.candidates.len(), 0); assert!(descriptor.capabilities.is_none()); assert!(descriptor.is_empty()); } diff --git a/shared/src/utils/hash_utils/tests/hashing_ids.rs b/shared/src/utils/hash_utils/tests/hashing_ids.rs index a02479b4e..570e79a37 100644 --- a/shared/src/utils/hash_utils/tests/hashing_ids.rs +++ b/shared/src/utils/hash_utils/tests/hashing_ids.rs @@ -34,7 +34,7 @@ fn numeric_id_with_trailing_slash() { fn non_numeric_last_segment_returns_hash() { let result = extract_id_from_url("http://server.com/live/user/pass/hello.ts"); assert!(!result.chars().all(|c| c.is_ascii_digit()), "should be a hash, not a numeric id"); - assert!(!result.is_empty()); + assert_ne!(result.len(), 0); } #[test] @@ -52,13 +52,13 @@ fn no_slash_returns_hash() { #[test] fn empty_url_returns_hash() { let result = extract_id_from_url(""); - assert!(!result.is_empty()); + assert_ne!(result.len(), 0); } #[test] fn only_slashes_returns_hash() { let result = extract_id_from_url("///"); - assert!(!result.is_empty()); + assert_ne!(result.len(), 0); } #[test] diff --git a/test/fixtures/hls/reduced-retry/baseline.m3u8 b/test/fixtures/hls/reduced-retry/baseline.m3u8 new file mode 100644 index 000000000..30da14cc1 --- /dev/null +++ b/test/fixtures/hls/reduced-retry/baseline.m3u8 @@ -0,0 +1,16 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:4 +#EXT-X-MEDIA-SEQUENCE:1000 +#EXTINF:4.0, +a1000.ts +#EXTINF:4.0, +a1001.ts +#EXTINF:4.0, +a1002.ts +#EXTINF:4.0, +a1003.ts +#EXTINF:4.0, +a1004.ts +#EXTINF:4.0, +a1005.ts diff --git a/test/fixtures/hls/reduced-retry/conflict.m3u8 b/test/fixtures/hls/reduced-retry/conflict.m3u8 new file mode 100644 index 000000000..0418d87c8 --- /dev/null +++ b/test/fixtures/hls/reduced-retry/conflict.m3u8 @@ -0,0 +1,16 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:4 +#EXT-X-MEDIA-SEQUENCE:1003 +#EXTINF:4.0, +b1003.ts +#EXTINF:4.0, +b1004.ts +#EXTINF:4.0, +b1005.ts +#EXTINF:4.0, +b1006.ts +#EXTINF:4.0, +b1007.ts +#EXTINF:4.0, +b1008.ts diff --git a/test/fixtures/hls/reduced-retry/repaired.m3u8 b/test/fixtures/hls/reduced-retry/repaired.m3u8 new file mode 100644 index 000000000..1e304d530 --- /dev/null +++ b/test/fixtures/hls/reduced-retry/repaired.m3u8 @@ -0,0 +1,16 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:4 +#EXT-X-MEDIA-SEQUENCE:1003 +#EXTINF:4.0, +a1003.ts +#EXTINF:4.0, +a1004.ts +#EXTINF:4.0, +a1005.ts +#EXTINF:4.0, +a1006.ts +#EXTINF:4.0, +a1007.ts +#EXTINF:4.0, +a1008.ts From 0c13a0d7016d0bba604747a6712c59b87155f041 Mon Sep 17 00:00:00 2001 From: euzu Date: Mon, 19 Oct 2026 15:43:54 +0000 Subject: [PATCH 2/2] ci: bump version v3.3.156 --- Cargo.lock | 42 +++++++++++++++++++++--------------------- Cargo.toml | 2 +- frontend/Cargo.toml | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f49ed90cb..9dbead061 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1292,7 +1292,7 @@ checksum = "431a4c31778fde52b4400de34975f219eeca55cc829a9de157cd743a5b230ecb" [[package]] name = "frontend" -version = "3.3.155" +version = "3.3.156" dependencies = [ "base64 0.23.1", "blake3", @@ -3795,7 +3795,7 @@ dependencies = [ [[package]] name = "shared" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "base64 0.23.1", @@ -4399,7 +4399,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tuliprox" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "arc-swap", @@ -4480,7 +4480,7 @@ dependencies = [ [[package]] name = "tuliprox-auth" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "axum", @@ -4503,7 +4503,7 @@ dependencies = [ [[package]] name = "tuliprox-btree" -version = "3.3.155" +version = "3.3.156" dependencies = [ "blake3", "crc32fast", @@ -4523,7 +4523,7 @@ dependencies = [ [[package]] name = "tuliprox-config-loader" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "blake3", @@ -4543,7 +4543,7 @@ dependencies = [ [[package]] name = "tuliprox-core" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "arc-swap", @@ -4594,7 +4594,7 @@ dependencies = [ [[package]] name = "tuliprox-curation" -version = "3.3.155" +version = "3.3.156" dependencies = [ "indexmap", "log", @@ -4611,7 +4611,7 @@ dependencies = [ [[package]] name = "tuliprox-dvr" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "blake3", @@ -4639,7 +4639,7 @@ dependencies = [ [[package]] name = "tuliprox-hls" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "async-compression", @@ -4676,7 +4676,7 @@ dependencies = [ [[package]] name = "tuliprox-iptv" -version = "3.3.155" +version = "3.3.156" dependencies = [ "base64 0.23.1", "bytes", @@ -4701,7 +4701,7 @@ dependencies = [ [[package]] name = "tuliprox-library" -version = "3.3.155" +version = "3.3.156" dependencies = [ "blake3", "bytes", @@ -4728,7 +4728,7 @@ dependencies = [ [[package]] name = "tuliprox-media-server" -version = "3.3.155" +version = "3.3.156" dependencies = [ "bytes", "futures", @@ -4747,7 +4747,7 @@ dependencies = [ [[package]] name = "tuliprox-messaging" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "chrono", @@ -4769,7 +4769,7 @@ dependencies = [ [[package]] name = "tuliprox-metadata" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "chrono", @@ -4792,7 +4792,7 @@ dependencies = [ [[package]] name = "tuliprox-mpegts" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "bytes", @@ -4806,7 +4806,7 @@ dependencies = [ [[package]] name = "tuliprox-parser" -version = "3.3.155" +version = "3.3.156" dependencies = [ "chrono", "chrono-tz", @@ -4826,7 +4826,7 @@ dependencies = [ [[package]] name = "tuliprox-processing" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "chrono", @@ -4865,7 +4865,7 @@ dependencies = [ [[package]] name = "tuliprox-repository" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "base64 0.23.1", @@ -4903,7 +4903,7 @@ dependencies = [ [[package]] name = "tuliprox-session" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "axum", @@ -4932,7 +4932,7 @@ dependencies = [ [[package]] name = "tuliprox-testkit" -version = "3.3.155" +version = "3.3.156" dependencies = [ "axum", "base64 0.23.1", diff --git a/Cargo.toml b/Cargo.toml index 572379895..cb44676c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["backend/*", "frontend", "shared", "tools/tuliprox-testkit"] resolver = "2" [workspace.package] -version = "3.3.155" +version = "3.3.156" edition = "2021" rust-version = "1.95.0" diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index a3ffe5483..3788a3ba0 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -9,7 +9,7 @@ workspace = true [dependencies] blake3.workspace = true -shared = { version = "3.3.155", path = "../shared" } +shared = { version = "3.3.156", path = "../shared" } chrono.workspace = true yew = "0.23.0" yew-router = "0.20.0"