From 4ff37967270e552f67efe9f6ca0b95b4deceeeb1 Mon Sep 17 00:00:00 2001 From: euzu Date: Mon, 14 Sep 2026 13:34:32 +0200 Subject: [PATCH] test-kit --- .github/workflows/ci.yaml | 40 + .gitignore | 3 + Cargo.lock | 28 + Cargo.toml | 2 +- Makefile | 23 + backend/app/src/api/api_utils/mod.rs | 90 +- backend/app/src/api/api_utils/tests.rs | 149 +- .../api/model/streams/active_client_stream.rs | 26 +- backend/app/src/api/model/streams/mod.rs | 1 + .../src/api/model/streams/provider_stream.rs | 13 +- .../model/streams/provider_stream_factory.rs | 211 +- backend/app/src/api/panel_api/mod.rs | 2 + backend/core/src/model/provider.rs | 62 +- backend/parser/src/m3u.rs | 2 + .../session/src/active_provider_manager.rs | 1327 +++++-- .../session/src/active_user_manager/mod.rs | 2 +- backend/session/src/admission_strategy.rs | 15 +- backend/session/src/stream.rs | 4 + backend/session/src/streams/mod.rs | 3 +- .../src/streams/provider_body_owner.rs | 375 ++ bin/run-testkit-scenarios.sh | 110 + docker/Dockerfile | 8 +- docker/cross.Dockerfile | 8 +- .../testkit/custom-video-fixtures.json | 12 + .../scenarios/evict-same-ip-latest.yml | 62 + .../scenarios/evict-same-ip-oldest.yml | 26 + .../testkit/scenarios/evict-user-latest.yml | 26 + .../testkit/scenarios/evict-user-oldest.yml | 26 + .../fixtures/testkit/scenarios/hard-limit.yml | 47 + .../testkit/scenarios/provider-hard-limit.yml | 38 + .../shared-stream-single-provider-slot.yml | 38 + .../vod-range-reopen-preserves-three-live.yml | 99 + .../scenarios/vod-range-reopen-strict-cap.yml | 94 + .../vod-reopen-backpressured-body.yml | 65 + tools/tuliprox-testkit/Cargo.toml | 35 + tools/tuliprox-testkit/src/adaptive.rs | 80 + tools/tuliprox-testkit/src/bootstrap.rs | 462 +++ tools/tuliprox-testkit/src/config.rs | 607 ++++ tools/tuliprox-testkit/src/control.rs | 63 + tools/tuliprox-testkit/src/custom_video.rs | 124 + tools/tuliprox-testkit/src/discovery.rs | 78 + tools/tuliprox-testkit/src/faults.rs | 50 + tools/tuliprox-testkit/src/frame.rs | 251 ++ tools/tuliprox-testkit/src/hls.rs | 124 + tools/tuliprox-testkit/src/lib.rs | 57 + tools/tuliprox-testkit/src/main.rs | 3107 +++++++++++++++++ tools/tuliprox-testkit/src/observation.rs | 49 + tools/tuliprox-testkit/src/oracle.rs | 237 ++ tools/tuliprox-testkit/src/origin_events.rs | 127 + .../tuliprox-testkit/src/origin_transport.rs | 484 +++ tools/tuliprox-testkit/src/policy.rs | 265 ++ tools/tuliprox-testkit/src/protocol.rs | 235 ++ tools/tuliprox-testkit/src/report.rs | 146 + tools/tuliprox-testkit/src/scheduler.rs | 85 + tools/tuliprox-testkit/src/secret.rs | 20 + tools/tuliprox-testkit/src/shared.rs | 53 + tools/tuliprox-testkit/src/transport.rs | 372 ++ tools/tuliprox-testkit/src/vod.rs | 345 ++ 58 files changed, 10129 insertions(+), 364 deletions(-) create mode 100644 backend/session/src/streams/provider_body_owner.rs create mode 100755 bin/run-testkit-scenarios.sh create mode 100644 test/fixtures/testkit/custom-video-fixtures.json create mode 100644 test/fixtures/testkit/scenarios/evict-same-ip-latest.yml create mode 100644 test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml create mode 100644 test/fixtures/testkit/scenarios/evict-user-latest.yml create mode 100644 test/fixtures/testkit/scenarios/evict-user-oldest.yml create mode 100644 test/fixtures/testkit/scenarios/hard-limit.yml create mode 100644 test/fixtures/testkit/scenarios/provider-hard-limit.yml create mode 100644 test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml create mode 100644 test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml create mode 100644 test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml create mode 100644 test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml create mode 100644 tools/tuliprox-testkit/Cargo.toml create mode 100644 tools/tuliprox-testkit/src/adaptive.rs create mode 100644 tools/tuliprox-testkit/src/bootstrap.rs create mode 100644 tools/tuliprox-testkit/src/config.rs create mode 100644 tools/tuliprox-testkit/src/control.rs create mode 100644 tools/tuliprox-testkit/src/custom_video.rs create mode 100644 tools/tuliprox-testkit/src/discovery.rs create mode 100644 tools/tuliprox-testkit/src/faults.rs create mode 100644 tools/tuliprox-testkit/src/frame.rs create mode 100644 tools/tuliprox-testkit/src/hls.rs create mode 100644 tools/tuliprox-testkit/src/lib.rs create mode 100644 tools/tuliprox-testkit/src/main.rs create mode 100644 tools/tuliprox-testkit/src/observation.rs create mode 100644 tools/tuliprox-testkit/src/oracle.rs create mode 100644 tools/tuliprox-testkit/src/origin_events.rs create mode 100644 tools/tuliprox-testkit/src/origin_transport.rs create mode 100644 tools/tuliprox-testkit/src/policy.rs create mode 100644 tools/tuliprox-testkit/src/protocol.rs create mode 100644 tools/tuliprox-testkit/src/report.rs create mode 100644 tools/tuliprox-testkit/src/scheduler.rs create mode 100644 tools/tuliprox-testkit/src/secret.rs create mode 100644 tools/tuliprox-testkit/src/shared.rs create mode 100644 tools/tuliprox-testkit/src/transport.rs create mode 100644 tools/tuliprox-testkit/src/vod.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bceab7a3d..5d0a13c74 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -140,3 +140,43 @@ jobs: - name: Run Tests run: make test + + admission: + name: Admission & Provider Lifecycle + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Stable Rust + uses: dtolnay/rust-toolchain@stable + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Verify admission and provider lifecycle invariants + run: make admission-test + + testkit: + name: Testkit Unit & E2E Scenarios + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Stable Rust + uses: dtolnay/rust-toolchain@stable + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Run Testkit Unit Tests + run: make testkit-test + + - name: Run Testkit E2E Scenarios + run: make testkit-e2e + + - name: Upload Testkit Reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: testkit-report + path: testkit-report/ diff --git a/.gitignore b/.gitignore index 7c343e3e7..f0022e80d 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ backend/app/Cargo.lock .env.* !.env.example +# Testkit report artifacts +/testkit-report/ + diff --git a/Cargo.lock b/Cargo.lock index b8b0c3eeb..e1206b254 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4896,6 +4896,34 @@ dependencies = [ "url", ] +[[package]] +name = "tuliprox-testkit" +version = "3.3.109" +dependencies = [ + "axum", + "base64 0.23.1", + "blake3", + "bytes", + "clap", + "crc32fast", + "futures", + "hyper", + "hyper-util", + "pin-project-lite", + "reqwest", + "serde", + "serde-saphyr", + "serde_json", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tokio-tungstenite", + "tower", + "tower-service", + "url", +] + [[package]] name = "tungstenite" version = "0.29.0" diff --git a/Cargo.toml b/Cargo.toml index 323de1c61..40f6549ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["backend/*", "frontend", "shared"] +members = ["backend/*", "frontend", "shared", "tools/tuliprox-testkit"] resolver = "2" [workspace.package] diff --git a/Makefile b/Makefile index 3a79d56a5..e714f9d9a 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,27 @@ test: ## Run all workspace tests (Stable) — use detected CPU count for paralle @echo "==> Running tests (stable) with $(CPU_COUNT) jobs/threads" @TMPDIR="$${TMPDIR:-/tmp}" RUST_TEST_THREADS=$(CPU_COUNT) ./bin/test.sh -j$(CPU_COUNT) --workspace -- --test-threads=$(CPU_COUNT) +.PHONY: admission-test +admission-test: ## Verify admission, provider-slot, seek/reopen, shared-stream, and cleanup invariants + @echo "==> Running HTTP admission and provider lifecycle tests" + @$(CARGO_STABLE) test --package tuliprox api_utils::tests:: -- --test-threads=1 + @echo "==> Running client stream lifecycle tests" + @$(CARGO_STABLE) test --package tuliprox api::model::streams::active_client_stream::tests:: -- --test-threads=1 + @echo "==> Running session, eviction, and provider-slot lifecycle tests" + @$(CARGO_STABLE) test --package tuliprox-session -- --test-threads=1 + +.PHONY: testkit-test +testkit-test: ## Run unit tests in tuliprox-testkit + @echo "==> Running testkit unit tests" + @$(CARGO_STABLE) test --package tuliprox-testkit + +.PHONY: testkit-e2e +testkit-e2e: ## Run testkit E2E scenario suite against freshly built SUT + @echo "==> Building tuliprox and tuliprox-testkit" + @$(CARGO_STABLE) build --package tuliprox --package tuliprox-testkit + @echo "==> Executing testkit scenario suite" + @./bin/run-testkit-scenarios.sh + .PHONY: build build: ## Build the entire workspace in parallel using detected CPU count @echo "==> Building workspace with $(CARGO_BUILD_JOBS) jobs" @@ -180,6 +201,8 @@ fmt: ## Format all code using nightly rules (Compact) fmt-check: ## Check if code follows formatting rules (Nightly) @echo "==> Checking formatting (nightly)" $(CARGO_NIGHTLY) fmt --all -- --check + @echo "==> Checking git diff for whitespace errors" + git diff --check .PHONY: markdown-lint markdown-lint: ## Lint markdown files diff --git a/backend/app/src/api/api_utils/mod.rs b/backend/app/src/api/api_utils/mod.rs index 8fedd5250..0cf77662d 100644 --- a/backend/app/src/api/api_utils/mod.rs +++ b/backend/app/src/api/api_utils/mod.rs @@ -7,12 +7,13 @@ use crate::{ endpoints::xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext}, model::{ create_active_client_stream, create_channel_unavailable_stream, create_custom_video_stream_response, - create_provider_connections_exhausted_stream, create_provider_stream, - get_custom_stream_response_error_status, get_stream_response_with_headers, is_custom_video_stream_enabled, + create_provider_connections_exhausted_stream, get_custom_stream_response_error_status, + get_stream_response_with_headers, is_custom_video_stream_enabled, open_provider_stream_with_lifecycle, tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, PlaybackLeaseRef, ProviderAllocation, ProviderConfig, ProviderStreamCustomReason, ProviderStreamFactoryOptions, - ProviderStreamInfo, ProviderStreamState, SharedStreamCtx, SharedStreamManager, StreamAdmissionError, - StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, + ProviderStreamInfo, ProviderStreamOpenLifecycle, ProviderStreamState, SharedStreamCtx, SharedStreamManager, + StreamAdmissionError, StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, + UserSession, }, }, auth::Fingerprint, @@ -189,7 +190,11 @@ pub(crate) fn admission_failure_response( /// players receive an unambiguous error instead of an empty success body. fn stream_admission_rejected_response(error: StreamAdmissionError, username: &str) -> axum::response::Response { error!("Stream admission rejected for user {username}: {error:?}"); - StatusCode::SERVICE_UNAVAILABLE.into_response() + axum::response::Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .header("x-tuliprox-rejection", "admission_rejected") + .body(axum::body::Body::empty()) + .unwrap_or_else(|_| StatusCode::SERVICE_UNAVAILABLE.into_response()) } #[macro_export] @@ -1156,7 +1161,7 @@ fn create_unmapped_provider_stream(app_config: &AppConfig) -> ProviderStreamStat } } -fn acquire_stream_provider_handle( +async fn acquire_stream_provider_handle( app_state: &Arc, input: &ConfigInput, fingerprint: &Fingerprint, @@ -1167,14 +1172,18 @@ fn acquire_stream_provider_handle( match options.force_provider { Some(provider) => { // First try to stay on the exact pinned provider account without over-allocating. - if let Some(handle) = app_state.active_provider.acquire_exact_connection_with_lease_for_session( - provider, - &fingerprint.addr, - options.allow_provider_grace, - options.user_priority, - options.connection_kind, - lease, - ) { + if let Some(handle) = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session_await( + provider, + &fingerprint.addr, + options.allow_provider_grace, + options.user_priority, + options.connection_kind, + lease, + ) + .await + { Some(managed(handle)) } else if options.allow_forced_provider_fallback { debug_if_enabled!( @@ -1184,7 +1193,7 @@ fn acquire_stream_provider_handle( ); app_state .active_provider - .acquire_connection_with_lease_for_session( + .acquire_connection_with_lease_for_session_await( &input.name, &fingerprint.addr, options.allow_provider_grace, @@ -1192,6 +1201,7 @@ fn acquire_stream_provider_handle( options.connection_kind, lease, ) + .await .map(managed) } else { debug_if_enabled!( @@ -1204,7 +1214,7 @@ fn acquire_stream_provider_handle( } None => app_state .active_provider - .acquire_connection_with_lease_for_session( + .acquire_connection_with_lease_for_session_await( &input.name, &fingerprint.addr, options.allow_provider_grace, @@ -1212,6 +1222,7 @@ fn acquire_stream_provider_handle( options.connection_kind, lease, ) + .await .map(managed), } } @@ -1269,7 +1280,7 @@ async fn resolve_streaming_strategy( ) -> StreamingStrategy { // allocate a provider connection let accept_requested_stream_url = options.accept_requested_stream_url || input.input_type.is_stalker(); - let mut provider_connection_handle = acquire_stream_provider_handle(app_state, input, fingerprint, &options); + let mut provider_connection_handle = acquire_stream_provider_handle(app_state, input, fingerprint, &options).await; // panel_api provisioning/loading is handled later in the stream creation flow @@ -1556,7 +1567,7 @@ async fn create_stream_response_details( match streaming_strategy.provider_stream_state { // custom stream means we display our own stream like connection exhausted, channel-unavailable... - ProviderStreamState::Custom { response: provider_stream, .. } => { + ProviderStreamState::Custom { response: provider_stream, reason } => { let (stream, stream_info) = provider_stream; // When allocation is exhausted or no connection was acquired, guard_provider_name is None. // Use input.name as fallback so the provider field is never empty. @@ -1577,6 +1588,7 @@ async fn create_stream_response_details( provider_handle: streaming_strategy.provider_handle.take(), content_representation, grace_resolution_context, + custom_reason: Some(reason), }) } ProviderStreamState::Available(_provider_name, request_url) @@ -1654,10 +1666,15 @@ async fn create_stream_response_details( } let reconnect_flag = provider_stream_factory_options.get_reconnect_flag_clone(); - let provider_stream = match create_provider_stream( + let lifecycle = streaming_strategy + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); + let provider_stream = match open_provider_stream_with_lifecycle( &app_state.provider_stream_ctx(), &app_state.http_client.load(), provider_stream_factory_options, + lifecycle, ) .await { @@ -1717,11 +1734,19 @@ async fn create_stream_response_details( } options.set_provider(input.get_resolve_provider(url.as_ref())); options.require_public_destination(); + if let Some(m) = streaming_strategy.provider_handle.as_mut() { + let _ = m.renew_opening_tokens(); + } let retry_reconnect_flag = options.get_reconnect_flag_clone(); - let retried = create_provider_stream( + let retry_lifecycle = streaming_strategy + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); + let retried = open_provider_stream_with_lifecycle( &app_state.provider_stream_ctx(), &app_state.http_client.load(), options, + retry_lifecycle, ) .await; if let Some(response) = retried { @@ -1790,6 +1815,7 @@ async fn create_stream_response_details( provider_handle, content_representation, grace_resolution_context, + custom_reason: None, }) } } @@ -2086,7 +2112,7 @@ pub async fn force_provider_stream_response( sanitize_sensitive_info(&user_session.token), sanitize_sensitive_info(&fingerprint.addr.to_string()) ); - cleanup_forced_reopen_addrs(app_state, &user_session.token, &cleanup_addrs); + cleanup_forced_reopen_addrs(app_state, &user_session.token, &cleanup_addrs).await; } // Provider-affine playback must stay on the same provider account across seeks/range reconnects. @@ -2742,6 +2768,24 @@ pub(crate) async fn stream_response( activation.placeholder_transition_version.is_some(), ) .await; + if stream_details.custom_reason == Some(ProviderStreamCustomReason::ProviderExhausted) { + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel, + provider_name: stream_details.provider_name.unwrap_or_else(|| input.name.clone()), + req_headers, + reason: ConnectFailureReason::ProviderConnectionsExhausted, + failure_stage: FailureStage::Admission, + }); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::ProviderConnectionsExhausted, + ) + .into_response(); + } no_custom_video_fallback_status(&app_state.app_config).into_response() } @@ -2991,8 +3035,8 @@ pub(crate) fn get_hls_session_ttl_secs(app_state: &Arc) -> u64 { get_stream_config_u64(app_state, |stream| stream.hls_session_ttl_secs, default_hls_session_ttl_secs()) } -fn cleanup_forced_reopen_addrs(app_state: &Arc, session_owner: &str, cleanup_addrs: &[SocketAddr]) { - app_state.active_provider.release_playback_connections(session_owner, cleanup_addrs); +async fn cleanup_forced_reopen_addrs(app_state: &Arc, session_owner: &str, cleanup_addrs: &[SocketAddr]) { + app_state.active_provider.release_playback_connections_await(session_owner, cleanup_addrs).await; } pub(crate) fn get_catchup_session_ttl_secs(app_state: &Arc) -> u64 { diff --git a/backend/app/src/api/api_utils/tests.rs b/backend/app/src/api/api_utils/tests.rs index ae0fa24cf..3be6e51b7 100644 --- a/backend/app/src/api/api_utils/tests.rs +++ b/backend/app/src/api/api_utils/tests.rs @@ -7200,6 +7200,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; assert!( should_pin_provider_for_session(&no_video_details, &app_state, PlaylistItemType::Catchup), @@ -7222,6 +7223,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; assert!( should_pin_provider_for_session(&provisioning_details, &app_state, PlaylistItemType::Catchup), @@ -7251,6 +7253,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; assert!( !should_pin_provider_for_session(&failure_details, &app_state, PlaylistItemType::Catchup), @@ -8276,7 +8279,7 @@ async fn forced_reopen_cleanup_for_adaptive_streams_does_not_close_client_socket let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!()); let mut close_rx = app_state.connection_manager.get_close_connection_channel(); - cleanup_forced_reopen_addrs(&app_state, "adaptive-owner", &[addr]); + cleanup_forced_reopen_addrs(&app_state, "adaptive-owner", &[addr]).await; let signal = tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); @@ -8289,7 +8292,7 @@ async fn forced_reopen_cleanup_for_non_adaptive_streams_preserves_client_socket( let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!()); let mut close_rx = app_state.connection_manager.get_close_connection_channel(); - cleanup_forced_reopen_addrs(&app_state, "live-owner", &[addr]); + cleanup_forced_reopen_addrs(&app_state, "live-owner", &[addr]).await; let signal = tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); @@ -8693,3 +8696,145 @@ fn evaluate_network_access_respects_allow_policy() { NetworkAccessDecision::AllowedGeoIpUnavailable ); } + +#[tokio::test] +async fn test_channel_unavailable_fallback_does_not_register_body_owner_or_leak_slot() { + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let mut cfg = (**app_state.app_config.config.load()).clone(); + cfg.custom_stream_response_enabled = true; + app_state.app_config.config.store(std::sync::Arc::new(cfg)); + let addr: std::net::SocketAddr = "127.0.0.1:49080".parse().unwrap(); + + let custom_video = crate::model::CustomStreamResponse { + channel_unavailable: Some(crate::api::model::TransportStreamBuffer::new(b"channel-unavailable-bytes".to_vec())), + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }; + app_state.app_config.custom_stream_response.store(Some(std::sync::Arc::new(custom_video))); + + let handle = app_state + .active_provider + .acquire_connection(&provider_name, &addr, 0, tuliprox_session::ConnectionKind::Normal) + .expect("connection should be acquired"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + let (channel_unavail_stream, info) = + create_channel_unavailable_stream(&app_state.app_config, &[], axum::http::StatusCode::OK); + assert!(channel_unavail_stream.is_some()); + + let factory_response = tuliprox_session::ProviderStreamFactoryResponse { + stream: channel_unavail_stream.unwrap(), + info, + provider_session_headers: std::collections::HashMap::new(), + has_upstream_owner: false, + }; + + if factory_response.has_upstream_owner { + app_state.active_provider.register_body_owner(handle.allocation_id); + } + + app_state.active_provider.release_handle(&handle); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::()).unwrap_or(0), 0); +} + +#[tokio::test] +async fn test_provider_open_cancelled_during_header_wait_releases_slot_without_leak() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + + let accepted = std::sync::Arc::new(tokio::sync::Notify::new()); + let accepted_clone = std::sync::Arc::clone(&accepted); + + let server_task = tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + accepted_clone.notify_waiters(); + let mut buf = [0u8; 1024]; + loop { + use tokio::io::AsyncReadExt; + match socket.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + } + }); + + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let client_addr: std::net::SocketAddr = "127.0.0.1:49081".parse().unwrap(); + + let handle = app_state + .active_provider + .acquire_connection(&provider_name, &client_addr, 0, tuliprox_session::ConnectionKind::Normal) + .expect("connection should be acquired"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + let url = url::Url::parse(&format!("http://{local_addr}/stream.ts")).unwrap(); + let req_headers = axum::http::HeaderMap::new(); + let stream_options = get_stream_options(&app_state.app_config); + let mut options = + crate::api::model::ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { + addr: client_addr, + item_type: PlaylistItemType::Live, + share_stream: false, + stream_options: &stream_options, + stream_url: &url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: tuliprox_session::ProviderContentRepresentationMode::PreserveOrigin, + }); + options.set_provider_handle_tokens( + handle.cancel_token.clone(), + handle.completion_token.clone(), + Some(handle.close_reason.clone()), + ); + app_state.active_provider.mark_opening(handle.allocation_id); + + let ctx = app_state.provider_stream_ctx(); + let client = app_state.http_client.load().as_ref().clone(); + + let open_task = + tokio::spawn(async move { crate::api::model::create_provider_stream(&ctx, &client, options).await }); + + // Wait until TCP connection is accepted and waiting for response headers + accepted.notified().await; + + // Now cancel the handle while it is waiting for response headers + handle.cancel_token.as_ref().unwrap().cancel(); + + // The open task should finish quickly because it is cancelled + let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), open_task) + .await + .expect("open task should abort upon cancellation") + .expect("task join"); + assert!(outcome.is_none()); + + // Release the handle + app_state.active_provider.release_handle(&handle); + + // Yield to allow background completion tasks to process + tokio::task::yield_now().await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::()).unwrap_or(0), 0); + + let _ = server_task.await; +} diff --git a/backend/app/src/api/model/streams/active_client_stream.rs b/backend/app/src/api/model/streams/active_client_stream.rs index 90fb4d19b..ccf5d385c 100644 --- a/backend/app/src/api/model/streams/active_client_stream.rs +++ b/backend/app/src/api/model/streams/active_client_stream.rs @@ -2,10 +2,10 @@ use crate::{ api::{ model::{ connection_manager::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET}, - create_provider_stream, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, CleanupEvent, - ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream, PendingProviderWakeSource, - ProviderStreamFactoryOptions, StreamDetails, StreamError, StreamMeterHandle, TimedClientStream, - TransportStreamBuffer, + open_provider_stream_with_lifecycle, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, + CleanupEvent, ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream, + PendingProviderWakeSource, ProviderStreamFactoryOptions, ProviderStreamOpenLifecycle, StreamDetails, + StreamError, StreamMeterHandle, TimedClientStream, TransportStreamBuffer, }, panel_api::{can_provision_on_exhausted, find_input_by_provider_name, run_panel_api_provisioning_probe}, }, @@ -421,6 +421,12 @@ impl ActiveClientStreamState { self.preempt_cancelled = None; self.stop_grace_task(); + let is_superseded = self + .provider_handle + .as_ref() + .and_then(|m| m.handle()) + .is_some_and(|h| h.get_close_reason() == tuliprox_core::model::ProviderCloseReason::Superseded); + let mut serve_preempted_custom = false; if self.provider_handle.is_some() { let managed = self.provider_handle.take(); @@ -428,7 +434,7 @@ impl ActiveClientStreamState { // Synchronous provider-slot release; the custom-video detail update is a // separate, best-effort UI effect. drop(managed); - if self.custom_video.low_priority_preempted.is_some() { + if !is_superseded && self.custom_video.low_priority_preempted.is_some() { serve_preempted_custom = true; if let Some(flag) = &self.send_custom_stream_flag { flag.store(StreamMode::LowPriorityPreempted as u8, Ordering::Release); @@ -706,11 +712,17 @@ impl Stream for ActiveClientStream { let http_client = app_state.http_client.load(); http_client.as_ref().clone() }; + let lifecycle = self + .state + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); let future = Box::pin(async move { - match create_provider_stream( + match open_provider_stream_with_lifecycle( &app_state.provider_stream_ctx(), &client, context.provider_stream_factory_options, + lifecycle, ) .await { @@ -1995,6 +2007,7 @@ mod tests { provider_handle: Some(provider_handle), content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, } } @@ -2934,6 +2947,7 @@ mod tests { provider_handle: None, content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: Some(grace_context.clone()), + custom_reason: None, }; let (flag, grace_task) = stream_grace_period(GracePeriodParams { diff --git a/backend/app/src/api/model/streams/mod.rs b/backend/app/src/api/model/streams/mod.rs index e7d52e732..500429096 100644 --- a/backend/app/src/api/model/streams/mod.rs +++ b/backend/app/src/api/model/streams/mod.rs @@ -45,4 +45,5 @@ pub use crate::utils::network::request::STREAM_IDLE_TIMEOUT; pub(crate) use shared::model::CustomVideoStreamType; // Shared-stream fan-out and its buffer moved to `tuliprox-session`; // re-exported so api call sites keep their names, module paths included. +#[cfg(test)] pub(crate) use tuliprox_session::streams::buffered_stream; diff --git a/backend/app/src/api/model/streams/provider_stream.rs b/backend/app/src/api/model/streams/provider_stream.rs index c1b912102..a5be0eab8 100644 --- a/backend/app/src/api/model/streams/provider_stream.rs +++ b/backend/app/src/api/model/streams/provider_stream.rs @@ -230,7 +230,18 @@ pub fn create_custom_video_stream_response( // configured `custom_stream_response_error_status` (default 502) instead of a // hard-coded 403, so a reverse proxy with `proxy_intercept_errors on;` can sever // the socket. - get_custom_stream_response_error_status(config).into_response() + let status = get_custom_stream_response_error_status(config); + let mut builder = axum::response::Response::builder().status(status); + if matches!( + video_response, + CustomVideoStreamType::UserConnectionsExhausted + | CustomVideoStreamType::ProviderConnectionsExhausted + | CustomVideoStreamType::LowPriorityPreempted + | CustomVideoStreamType::UserAccountExpired + ) { + builder = builder.header("x-tuliprox-rejection", "admission_rejected"); + } + builder.body(axum::body::Body::empty()).unwrap_or_else(|_| status.into_response()) } pub fn get_header_filter_for_item_type(item_type: PlaylistItemType) -> HeaderFilter { match item_type { diff --git a/backend/app/src/api/model/streams/provider_stream_factory.rs b/backend/app/src/api/model/streams/provider_stream_factory.rs index a7ac2b4ad..e22003c89 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory.rs @@ -1,9 +1,8 @@ use crate::{ api::model::{ create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers, - streams::{buffered_stream::BufferedStream, client_stream::ClientStream}, - CustomVideoStreamType, ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError, - STREAM_IDLE_TIMEOUT, + streams::client_stream::ClientStream, CustomVideoStreamType, ProviderContentRepresentationMode, + ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT, }, iptv::stalker::client::validate_public_playable_url, model::{AppConfig, ConfigProvider, ReverseProxyDisabledHeaderConfig}, @@ -36,11 +35,11 @@ use std::{ error::Error as StdError, io, net::SocketAddr, - sync::Arc, + sync::{atomic::AtomicU8, Arc}, time::{Duration, Instant}, }; use tokio_util::{io::ReaderStream, sync::CancellationToken}; -use tuliprox_core::utils::request_headers::get_headers_from_request; +use tuliprox_core::{model::AllocationId, utils::request_headers::get_headers_from_request}; use tuliprox_hls::api::{ extract_hls_provider_session_headers, log_hls_origin_content_coding, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource, @@ -49,6 +48,8 @@ use tuliprox_session::{ response_headers::{provider_response_headers, ProviderResponseHeaderError}, stream_ctx::ProviderStreamCtx, stream_options::StreamOptions, + streams::{ProviderBodyOwner, ProviderBodyOwnerConfig}, + ActiveProviderManager, ManagedProviderHandle, }; use url::Url; @@ -103,6 +104,9 @@ pub struct ProviderStreamFactoryOptions { content_representation: ProviderContentRepresentationMode, response_head_availability: ProviderResponseHeadAvailability, hls_content_coding_object_kind: Option, + cancel_token: Option, + completion_token: Option, + close_reason: Option>, } pub(crate) struct ProviderStreamFactoryParams<'a> { @@ -217,9 +221,29 @@ impl ProviderStreamFactoryOptions { ProviderContentRepresentationMode::Identity => Some(HlsOriginContentCodingObjectKind::Other), ProviderContentRepresentationMode::PreserveOrigin => None, }, + cancel_token: None, + completion_token: None, + close_reason: None, } } + pub fn set_provider_handle_tokens( + &mut self, + cancel_token: Option, + completion_token: Option, + close_reason: Option>, + ) { + self.cancel_token = cancel_token; + self.completion_token = completion_token; + self.close_reason = close_reason; + } + + pub fn get_cancel_token(&self) -> Option { self.cancel_token.clone() } + + pub fn get_completion_token(&self) -> Option { self.completion_token.clone() } + + pub fn get_close_reason(&self) -> Option> { self.close_reason.clone() } + pub fn set_provider(&mut self, provider: Option>) { self.provider = provider; } pub fn apply_user_agent_stream_index(&mut self, stream_index: u64) { @@ -748,7 +772,12 @@ async fn prepare_provider_stream_response_with_context( let headers = provider_response_headers(response.headers(), context.representation)?; let response_info = Some((headers, response.status(), Some(response.url().clone()), None)); let stream = response.bytes_stream().map_err(|error| StreamError::reqwest(&error)).boxed(); - Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers }) + Ok(ProviderStreamFactoryResponse { + stream, + info: response_info, + provider_session_headers, + has_upstream_owner: false, + }) } ProviderContentRepresentationMode::Identity => { let origin_status = response.status(); @@ -787,7 +816,12 @@ async fn prepare_provider_stream_response_with_context( let headers = provider_response_headers(&decoded.headers, context.representation)?; let response_info = Some((headers, decoded.status, Some(decoded.final_url), None)); let stream = ReaderStream::new(decoded.body).map_err(|error| provider_decoded_body_error(&error)).boxed(); - Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers }) + Ok(ProviderStreamFactoryResponse { + stream, + info: response_info, + provider_session_headers, + has_upstream_owner: false, + }) } } } @@ -963,6 +997,7 @@ async fn provider_stream_request( } } +#[allow(clippy::too_many_lines)] async fn get_provider_stream( ctx: &ProviderStreamCtx, client: &reqwest::Client, @@ -973,8 +1008,29 @@ async fn get_provider_stream( let start = Instant::now(); let mut connect_err: u32 = 1; + let handle_cancel = stream_options.get_cancel_token(); while stream_options.should_continue() { - match provider_stream_request(ctx, client, stream_options).await { + let request_future = provider_stream_request(ctx, client, stream_options); + let request_res = if let Some(ref cancel) = handle_cancel { + tokio::select! { + biased; + () = cancel.cancelled() => { + debug!( + "Provider stream request cancelled during open: {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + return Err(ProviderStreamRequestFailure::Status { + status: StatusCode::BAD_GATEWAY, + provider_error_class: "cancelled", + serve_channel_unavailable: false, + }); + } + res = request_future => res, + } + } else { + request_future.await + }; + match request_res { Ok(Some(stream_response)) => { return Ok(Some(stream_response)); } @@ -1030,7 +1086,15 @@ async fn get_provider_stream( break; } connect_err += 1; - tokio::time::sleep(Duration::from_millis(50)).await; + if let Some(ref cancel) = handle_cancel { + tokio::select! { + biased; + () = cancel.cancelled() => break, + () = tokio::time::sleep(Duration::from_millis(50)) => {}, + } + } else { + tokio::time::sleep(Duration::from_millis(50)).await; + } debug_if_enabled!("Reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); } debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); @@ -1042,14 +1106,31 @@ async fn get_provider_stream( }) } +struct ProviderOpenGuard { + token: Option, + handed_off: bool, +} + +impl Drop for ProviderOpenGuard { + fn drop(&mut self) { + if !self.handed_off { + if let Some(token) = self.token.take() { + token.cancel(); + } + } + } +} + #[allow(clippy::too_many_lines)] pub async fn create_provider_stream( ctx: &ProviderStreamCtx, client: &reqwest::Client, stream_options: ProviderStreamFactoryOptions, ) -> Option { + let mut open_guard = ProviderOpenGuard { token: stream_options.get_completion_token(), handed_off: false }; match get_provider_stream(ctx, client, &stream_options).await { - Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers })) => { + Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers, .. })) => { + let continue_signal = stream_options.get_reconnect_flag_clone(); if let Some((_headers, _status, _response_url, Some(custom_video_type))) = &info { let reason = match custom_video_type { CustomVideoStreamType::ChannelUnavailable => Some(ConnectFailureReason::ChannelUnavailable), @@ -1062,26 +1143,66 @@ pub async fn create_provider_stream( if let Some(reason) = reason { record_provider_open_failure(ctx, &stream_options, reason, None, None); } + return Some(ProviderStreamFactoryResponse { + stream: ClientStream::new(init_stream, continue_signal, None, stream_options.get_url_as_str()) + .boxed(), + info, + provider_session_headers, + has_upstream_owner: false, + }); } - let continue_signal = stream_options.get_reconnect_flag_clone(); - let stream = init_stream.boxed(); - let stream = if should_wrap_provider_stream_in_buffer(&stream_options) { - BufferedStream::new( - stream, + let handle_cancel = stream_options.get_cancel_token(); + let completion_token = stream_options.get_completion_token(); + let close_reason = stream_options.get_close_reason(); + + let cancel_token = if let Some(hc) = handle_cancel { + let combined = CancellationToken::new(); + let c1 = combined.clone(); + let c2 = combined.clone(); + let hc_clone = hc.clone(); + let cs_clone = continue_signal.clone(); + tokio::spawn(async move { + tokio::select! { + biased; + () = hc_clone.cancelled() => { + cs_clone.cancel(); + c1.cancel(); + } + () = cs_clone.cancelled() => { + c2.cancel(); + } + } + }); + combined + } else { + continue_signal.clone() + }; + + let body_owner_config = if should_wrap_provider_stream_in_buffer(&stream_options) { + ProviderBodyOwnerConfig::for_buffered( stream_options.get_buffer_size(), stream_options.get_buffer_max_bytes(), - stream_options.get_reconnect_flag_clone(), - stream_options.get_url_as_str(), ) - .boxed() } else { - stream + ProviderBodyOwnerConfig::for_direct_body() }; + + open_guard.handed_off = true; + let stream = ProviderBodyOwner::new( + init_stream.boxed(), + body_owner_config, + cancel_token, + completion_token, + close_reason, + ) + .boxed(); + Some(ProviderStreamFactoryResponse { stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()) .boxed(), info, provider_session_headers, + has_upstream_owner: true, }) } Ok(None) => None, @@ -1103,6 +1224,7 @@ pub async fn create_provider_stream( stream: boxed_provider_stream, info: response_info, provider_session_headers: HashMap::new(), + has_upstream_owner: false, }); } None @@ -1110,6 +1232,57 @@ pub async fn create_provider_stream( } } +#[derive(Clone)] +pub struct ProviderStreamOpenLifecycle { + manager: Arc, + allocation_id: AllocationId, + cancel_token: Option, + completion_token: Option, + close_reason: Option>, +} + +impl ProviderStreamOpenLifecycle { + pub fn from_managed(managed: &ManagedProviderHandle) -> Option { + let handle = managed.handle()?; + Some(Self { + manager: Arc::clone(managed.manager()), + allocation_id: handle.allocation_id, + cancel_token: handle.cancel_token.clone(), + completion_token: handle.completion_token.clone(), + close_reason: Some(Arc::clone(&handle.close_reason)), + }) + } + + pub fn mark_opening(&self) { self.manager.mark_opening(self.allocation_id); } + + pub fn register_body_owner(&self) -> bool { self.manager.register_body_owner(self.allocation_id) } +} + +pub async fn open_provider_stream_with_lifecycle( + ctx: &ProviderStreamCtx, + client: &reqwest::Client, + mut stream_options: ProviderStreamFactoryOptions, + lifecycle: Option, +) -> Option { + if let Some(ref lc) = lifecycle { + lc.mark_opening(); + stream_options.set_provider_handle_tokens( + lc.cancel_token.clone(), + lc.completion_token.clone(), + lc.close_reason.clone(), + ); + } + let response = create_provider_stream(ctx, client, stream_options).await?; + if response.has_upstream_owner { + if let Some(ref lc) = lifecycle { + if !lc.register_body_owner() { + return None; + } + } + } + Some(response) +} + #[cfg(test)] mod tests { use super::*; diff --git a/backend/app/src/api/panel_api/mod.rs b/backend/app/src/api/panel_api/mod.rs index 23f610a70..ea3e8246a 100644 --- a/backend/app/src/api/panel_api/mod.rs +++ b/backend/app/src/api/panel_api/mod.rs @@ -3692,6 +3692,7 @@ pub fn create_panel_api_provisioning_stream_details( provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; } @@ -3722,6 +3723,7 @@ pub fn create_panel_api_provisioning_stream_details( provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, } } diff --git a/backend/core/src/model/provider.rs b/backend/core/src/model/provider.rs index 578c1eaed..ff9d79509 100644 --- a/backend/core/src/model/provider.rs +++ b/backend/core/src/model/provider.rs @@ -94,6 +94,42 @@ impl ProviderAllocation { } } +use std::sync::atomic::{AtomicU8, Ordering}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectionLifecycle { + Opening, + Active, + Closing, + Closed, +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderCloseReason { + Unspecified = 0, + Superseded = 1, + PriorityPreempted = 2, + ClientClosed = 3, + IdleTimeout = 4, + ProviderError = 5, + Shutdown = 6, +} + +impl ProviderCloseReason { + pub fn from_u8(v: u8) -> Self { + match v { + 1 => Self::Superseded, + 2 => Self::PriorityPreempted, + 3 => Self::ClientClosed, + 4 => Self::IdleTimeout, + 5 => Self::ProviderError, + 6 => Self::Shutdown, + _ => Self::Unspecified, + } + } +} + #[derive(Debug, Clone)] pub struct ProviderHandle { pub playback_request_id: Option, @@ -106,6 +142,8 @@ pub struct ProviderHandle { pub allocation: ProviderAllocation, // Token to cancel the background task (e.g. internal probe) if preempted pub cancel_token: Option, + pub completion_token: Option, + pub close_reason: Arc, } impl ProviderHandle { @@ -115,7 +153,29 @@ impl ProviderHandle { allocation: ProviderAllocation, cancel_token: Option, ) -> Self { - Self { client_id, allocation_id, allocation, cancel_token, playback_request_id: None, binding_tag: None } + Self { + client_id, + allocation_id, + allocation, + cancel_token, + completion_token: None, + close_reason: Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8)), + playback_request_id: None, + binding_tag: None, + } + } + + pub fn set_close_reason(&self, reason: ProviderCloseReason) { + let _ = self.close_reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + reason as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + + pub fn get_close_reason(&self) -> ProviderCloseReason { + ProviderCloseReason::from_u8(self.close_reason.load(Ordering::Acquire)) } } diff --git a/backend/parser/src/m3u.rs b/backend/parser/src/m3u.rs index e0095e47e..145d8cd39 100644 --- a/backend/parser/src/m3u.rs +++ b/backend/parser/src/m3u.rs @@ -187,6 +187,8 @@ fn classify_token(t: &str) -> M3uToken { 4 => { if eq_ascii(bytes, b"cuid") { M3uToken::ProviderId + } else if eq_ascii(bytes, b"type") { + M3uToken::TvgType } else { M3uToken::Unknown } diff --git a/backend/session/src/active_provider_manager.rs b/backend/session/src/active_provider_manager.rs index 66ac12fe0..97e937349 100644 --- a/backend/session/src/active_provider_manager.rs +++ b/backend/session/src/active_provider_manager.rs @@ -10,18 +10,18 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, net::SocketAddr, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}, Arc, LazyLock, OnceLock, RwLockReadGuard, RwLockWriteGuard, Weak, }, - time::Instant, + time::{Duration, Instant}, }; use tokio::time::Instant as TokioInstant; use tokio_util::sync::CancellationToken; use tuliprox_core::{ model::{ - AllocationId, AppConfig, ConfigInput, GracePeriodOptions, PlaybackKind, PlaybackLeaseId, PlaybackRequestId, - PlaybackRequestOutcome, PlaybackSelectionReason, ProviderAllocation, ProviderBindingTag, ProviderConfig, - ProviderHandle, SharedSubscriberId, + AllocationId, AppConfig, ConfigInput, ConnectionLifecycle, GracePeriodOptions, PlaybackKind, PlaybackLeaseId, + PlaybackRequestId, PlaybackRequestOutcome, PlaybackSelectionReason, ProviderAllocation, ProviderBindingTag, + ProviderCloseReason, ProviderConfig, ProviderHandle, SharedSubscriberId, }, utils::debug_if_enabled, }; @@ -37,6 +37,8 @@ type PreemptionCandidate = (PriorityOwner, AllocationId, i8, Instant); // (smallest `created_at`) sorts last and is evicted first. type PriorityKey = (i8, Reverse, AllocationId); +const PREEMPTION_COMPLETION_TIMEOUT: Duration = Duration::from_millis(1500); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ConnectionKind { Normal, @@ -134,6 +136,12 @@ struct ActiveConnectionInfo { allocation: ProviderAllocation, // Used to signal preemption to the consumer of this connection cancel_token: CancellationToken, + completion_token: CancellationToken, + close_reason: Arc, + lifecycle: ConnectionLifecycle, + has_body_owner: bool, + reaper_spawned: bool, + open_generation: u64, created_at: Instant, priority: i8, kind: ConnectionKind, @@ -187,6 +195,9 @@ impl std::fmt::Debug for ManagedProviderHandle { } impl ManagedProviderHandle { + #[inline] + pub fn manager(&self) -> &Arc { &self.manager } + #[inline] pub fn new(manager: Arc, handle: ProviderHandle) -> Self { Self { manager, handle: Some(handle) } @@ -200,6 +211,31 @@ impl ManagedProviderHandle { #[inline] pub fn disarm(&mut self) -> Option { self.handle.take() } + + pub fn mark_opening(&self) { + if let Some(handle) = self.handle.as_ref() { + self.manager.mark_opening(handle.allocation_id); + } + } + + pub fn register_body_owner(&self) -> bool { + if let Some(handle) = self.handle.as_ref() { + self.manager.register_body_owner(handle.allocation_id) + } else { + false + } + } + + pub fn renew_opening_tokens(&mut self) -> Option<(CancellationToken, CancellationToken)> { + if let Some(ref mut handle) = self.handle { + if let Some((cancel, completion)) = self.manager.renew_opening_tokens(handle.allocation_id) { + handle.cancel_token = Some(cancel.clone()); + handle.completion_token = Some(completion.clone()); + return Some((cancel, completion)); + } + } + None + } } impl Drop for ManagedProviderHandle { @@ -210,18 +246,18 @@ impl Drop for ManagedProviderHandle { } } -pub struct ActiveProviderManager { +pub struct ActiveProviderManagerCore { // Serializes capacity transitions across counters, allocation indices and leases. capacity_transition: std::sync::Mutex<()>, is_shutting_down: AtomicBool, + shutdown_token: CancellationToken, providers: ProviderLineupManager, connections: std::sync::RwLock, leases: std::sync::RwLock, next_allocation_id: AtomicU64, - shared_stream_manager: OnceLock>, } -impl ActiveProviderManager { +impl ActiveProviderManagerCore { fn lock_capacity_transition(&self) -> std::sync::MutexGuard<'_, ()> { match self.capacity_transition.lock() { Ok(guard) => guard, @@ -270,6 +306,133 @@ impl ActiveProviderManager { } } + pub fn complete_release(&self, alloc_id: AllocationId) { self.complete_release_with_generation(alloc_id, None); } + + pub fn complete_release_with_generation(&self, alloc_id: AllocationId, generation: Option) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + Self::complete_release_locked(&mut connections, alloc_id, generation); + } + + fn complete_release_locked( + connections: &mut Connections, + alloc_id: AllocationId, + expected_generation: Option, + ) -> Option { + if let Some(info) = connections.single.get(&alloc_id) { + if let Some(expected_gen) = expected_generation { + if info.open_generation != expected_gen { + return None; + } + } + } + if let Some(mut info) = connections.single.remove(&alloc_id) { + ActiveProviderManager::unindex_owner(connections, alloc_id, info.session_owner.as_ref()); + if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { + set.remove(&alloc_id); + if set.is_empty() { + connections.single_by_addr.remove(&info.client_addr); + } + } + if !info.allocation.is_unlimited_provider() { + if let Some(name) = info.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&alloc_id); + } + ActiveProviderManager::remove_priority_entry( + connections, + &name, + &(info.priority, Reverse(info.created_at), alloc_id), + info.kind, + ); + } + } + info.lifecycle = ConnectionLifecycle::Closed; + info.cancel_token.cancel(); + info.allocation.release(); + Some(info.allocation) + } else { + None + } + } + + fn plan_single_release_locked(connections: &mut Connections, alloc_id: AllocationId) -> ReleaseAction { + let Some(info) = connections.single.get_mut(&alloc_id) else { + return ReleaseAction::None; + }; + if info.lifecycle == ConnectionLifecycle::Closed { + return ReleaseAction::None; + } + + info.cancel_token.cancel(); + + let completed = info.completion_token.is_cancelled(); + let needs_completion_wait = (info.lifecycle == ConnectionLifecycle::Closing + || info.lifecycle == ConnectionLifecycle::Opening + || info.has_body_owner) + && !completed; + + if needs_completion_wait { + info.lifecycle = ConnectionLifecycle::Closing; + let client_addr = info.client_addr; + let prio = info + .allocation + .get_provider_name() + .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)); + let already_spawned = info.reaper_spawned; + info.reaper_spawned = true; + let gen = info.open_generation; + let completion_token = info.completion_token.clone(); + + if let Some((name, key, kind)) = prio { + ActiveProviderManager::remove_priority_entry(connections, &name, &key, kind); + } + if let Some(set) = connections.single_by_addr.get_mut(&client_addr) { + set.remove(&alloc_id); + if set.is_empty() { + connections.single_by_addr.remove(&client_addr); + } + } + if already_spawned { + ReleaseAction::None + } else { + ReleaseAction::Wait(alloc_id, gen, completion_token) + } + } else { + let gen = info.open_generation; + Self::complete_release_locked(connections, alloc_id, Some(gen)); + ReleaseAction::Immediate + } + } +} + +#[derive(Debug)] +enum ReleaseAction { + None, + Immediate, + Wait(AllocationId, u64, CancellationToken), +} + +#[derive(Debug)] +pub(crate) enum PreemptionOutcome { + Acquired(ProviderAllocation), + PendingCompletion(CancellationToken), + Exhausted, +} + +pub struct ActiveProviderManager { + core: Arc, + shared_stream_manager: OnceLock>, +} + +impl std::ops::Deref for ActiveProviderManager { + type Target = ActiveProviderManagerCore; + + #[inline] + fn deref(&self) -> &Self::Target { &self.core } +} + +impl ActiveProviderManager { fn upsert_priority_entry( connections: &mut Connections, provider_name: &Arc, @@ -336,12 +499,15 @@ impl ActiveProviderManager { let grace_period_options = Self::get_grace_options(cfg); let inputs = Self::get_config_inputs(cfg); Self { - capacity_transition: std::sync::Mutex::new(()), - is_shutting_down: AtomicBool::new(false), - providers: ProviderLineupManager::new(inputs, grace_period_options, event_manager), - connections: std::sync::RwLock::new(Connections::default()), - leases: std::sync::RwLock::new(ProviderLeaseTable::default()), - next_allocation_id: AtomicU64::new(1), + core: Arc::new(ActiveProviderManagerCore { + capacity_transition: std::sync::Mutex::new(()), + is_shutting_down: AtomicBool::new(false), + shutdown_token: CancellationToken::new(), + providers: ProviderLineupManager::new(inputs, grace_period_options, event_manager), + connections: std::sync::RwLock::new(Connections::default()), + leases: std::sync::RwLock::new(ProviderLeaseTable::default()), + next_allocation_id: AtomicU64::new(1), + }), shared_stream_manager: OnceLock::new(), } } @@ -352,6 +518,7 @@ impl ActiveProviderManager { /// Closes provider admission and releases all physical allocations and leases. pub fn shutdown(&self) { + self.shutdown_token.cancel(); self.is_shutting_down.store(true, Ordering::Release); let _transition = self.lock_capacity_transition(); let connections = std::mem::take(&mut *self.write_connections()); @@ -806,7 +973,7 @@ impl ActiveProviderManager { leases.lease_of_owner(owner).map(|lease| ProviderBindingTag::new(lease.id, lease.binding_generation)) } - fn acquire_exact_connection_inner( + fn acquire_exact_connection_inner_no_preempt( &self, provider_name: &Arc, allow_grace: bool, @@ -824,6 +991,27 @@ impl ActiveProviderManager { Some(self.register_allocation(allocation.take(), params)) } + fn acquire_exact_connection_inner( + &self, + provider_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, params) { + return Some(handle); + } + if let Some(preempted_alloc) = self.try_preempt_connection( + provider_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) { + return Some(self.register_allocation(preempted_alloc, params)); + } + None + } + fn finalize_lineup_allocation( &self, input_name: &Arc, @@ -863,7 +1051,9 @@ impl ActiveProviderManager { let mut victim = None; for (prov_name, tree) in index { - if !self.providers.is_provider_for_input(prov_name, input_name) || reserved_providers.contains(prov_name) { + if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) + || reserved_providers.contains(prov_name) + { continue; } @@ -894,6 +1084,7 @@ impl ActiveProviderManager { match candidate.0 { PriorityOwner::Single(alloc_id) => connections.single.get(&alloc_id).is_some_and(|info| { info.kind == ConnectionKind::Normal + && info.lifecycle != ConnectionLifecycle::Closing && info.allocation.get_provider_name().as_ref() == Some(provider_name) }), PriorityOwner::Shared(shared_id) => connections @@ -931,7 +1122,7 @@ impl ActiveProviderManager { let mut victim = None; for (prov_name, tree) in &connections.priority_index { - if !self.providers.is_provider_for_input(prov_name, input_name) + if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) || reserved_providers.contains(prov_name) { continue; @@ -963,22 +1154,15 @@ impl ActiveProviderManager { } } - fn acquire_connection_inner( + fn acquire_connection_inner_no_preempt( &self, provider_or_input_name: &Arc, allow_grace: bool, params: &AcquireProviderParams<'_>, ) -> Option { - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } if let Some(owner) = params.session_owner() { if let Some(reserved_provider) = self.get_reserved_provider_for_owner(provider_or_input_name, owner) { - return self.acquire_exact_connection_inner(&reserved_provider, allow_grace, params); + return self.acquire_exact_connection_inner_no_preempt(&reserved_provider, allow_grace, params); } } @@ -1008,6 +1192,26 @@ impl ActiveProviderManager { return Some(self.finalize_lineup_allocation(provider_or_input_name, allow_grace, allocation, params)); } + None + } + + fn acquire_connection_inner( + &self, + provider_or_input_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_connection_inner_no_preempt(provider_or_input_name, allow_grace, params) { + return Some(handle); + } + if let Some(preempted_alloc) = self.try_preempt_connection( provider_or_input_name, params.priority, @@ -1031,6 +1235,8 @@ impl ActiveProviderManager { let provider_name = allocation.get_provider_name().unwrap_or_default(); let allocation_id = self.next_allocation_id.fetch_add(1, Ordering::Relaxed); let cancel_token = CancellationToken::new(); + let completion_token = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8)); let now = Instant::now(); let is_unlimited = allocation.is_unlimited_provider(); @@ -1066,6 +1272,12 @@ impl ActiveProviderManager { client_addr: *addr, allocation: allocation.clone(), cancel_token: cancel_token.clone(), + completion_token: completion_token.clone(), + close_reason: Arc::clone(&close_reason), + lifecycle: ConnectionLifecycle::Active, + has_body_owner: false, + reaper_spawned: false, + open_generation: 0, created_at: now, priority, kind, @@ -1103,6 +1315,8 @@ impl ActiveProviderManager { sanitize_sensitive_info(&addr.to_string()) ); let mut handle = ProviderHandle::new(*addr, allocation_id, allocation, Some(cancel_token)); + handle.completion_token = Some(completion_token); + handle.close_reason = close_reason; handle.playback_request_id = lease.map(|lease| lease.request_id); handle.binding_tag = binding_tag; handle @@ -1178,70 +1392,236 @@ impl ActiveProviderManager { if let Some(token) = cancel_token { token.cancel(); } - allocation.release(); - // Stop the shared stream broadcast task to match the released capacity. - // Without this, the broadcast keeps running and consuming a provider slot - // that was already freed by allocation.release(). if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { tokio::spawn(async move { ssm.teardown_preempted_stream(&stream_url, alloc_id).await; + allocation.release(); }); + return false; } + allocation.release(); } } PriorityOwner::Single(victim_alloc_id) => { - let removed_info = { - let mut connections = self.write_connections(); - if let Some(info) = connections.single.get(&victim_alloc_id) { - if info.priority != v_prio || info.created_at != victim_created_at { - return false; - } + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get(&victim_alloc_id) { + if info.priority != v_prio || info.created_at != victim_created_at { + return false; } - if let Some(info) = connections.single.remove(&victim_alloc_id) { - debug_if_enabled!( - "Grace-evicting single connection from {} (prio={}) on input {} for higher priority request (prio={})", - sanitize_sensitive_info(&info.client_addr.to_string()), - v_prio, - sanitize_sensitive_info(input_name), - new_priority - ); - Self::unindex_owner(&mut connections, victim_alloc_id, info.session_owner.as_ref()); - if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { - set.remove(&victim_alloc_id); - if set.is_empty() { - connections.single_by_addr.remove(&info.client_addr); + } + if let Some(info) = connections.single.get_mut(&victim_alloc_id) { + debug_if_enabled!( + "Grace-evicting single connection from {} (prio={}) on input {} for higher priority request (prio={})", + sanitize_sensitive_info(&info.client_addr.to_string()), + v_prio, + sanitize_sensitive_info(input_name), + new_priority + ); + info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); + } + let action = ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); + if let ReleaseAction::Wait(alloc_id, gen, completion_token) = action { + let shutdown_token = self.shutdown_token.clone(); + let core = Arc::clone(&self.core); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); } } - if let Some(name) = info.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&victim_alloc_id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(v_prio, Reverse(victim_created_at), victim_alloc_id), - info.kind, - ); - } - Some(info) - } else { - None - } - }; - - let Some(info) = removed_info else { + }); return false; - }; - // Preempted probes must stop immediately; they must not keep a custom stream alive. - info.cancel_token.cancel(); - info.allocation.release(); + } } } true } + fn try_acquire_allocation_after_freed( + &self, + input_name: &Arc, + allow_grace: bool, + reserved_providers: &HashSet>, + session_owner: Option<&str>, + ) -> Option { + let attempts = self.providers.provider_names_for_input(input_name).len().max(1); + let mut excluded_providers = reserved_providers.clone(); + for _ in 0..attempts { + let allocation = self.providers.acquire_connection_with_grace_override_excluding( + input_name, + allow_grace, + &excluded_providers, + ); + if matches!(allocation, ProviderAllocation::Exhausted) { + break; + } + if let Some(provider_name) = allocation.get_provider_name() { + if self.exceeds_reserved_capacity(&provider_name, session_owner) { + excluded_providers.insert(provider_name); + allocation.release(); + continue; + } + } + return Some(allocation); + } + None + } + #[allow(clippy::too_many_lines)] + fn try_preempt_connection_outcome( + &self, + input_name: &Arc, + new_priority: i8, + allow_grace: bool, + kind_needed: ConnectionKind, + session_owner: Option<&str>, + ) -> PreemptionOutcome { + let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); + let victim = { + let connections = self.read_connections(); + self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) + }; + + let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else { + return PreemptionOutcome::Exhausted; + }; + + match owner { + PriorityOwner::Shared(shared_id) => { + debug_if_enabled!( + "Preempting shared connection (allocation_id={shared_id}, prio={v_prio}) for higher priority request (prio={new_priority})" + ); + let released_shared_allocation = { + let mut connections = self.write_connections(); + let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else { + return PreemptionOutcome::Exhausted; + }; + + let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { + shared.allocation_id == alloc_id + && shared.priority == v_prio + && shared.created_at == victim_created_at + }); + if !still_match { + return PreemptionOutcome::Exhausted; + } + + if let Some(shared) = connections.shared.by_key.remove(&key) { + connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); + Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); + } + + if let Some(name) = shared.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&shared.allocation_id); + } + Self::remove_priority_entry( + &mut connections, + &name, + &(v_prio, Reverse(victim_created_at), alloc_id), + shared.kind, + ); + } + Some((key, shared.allocation, shared.cancel_token)) + } else { + None + } + }; + + let Some((stream_url, allocation, cancel_token)) = released_shared_allocation else { + return PreemptionOutcome::Exhausted; + }; + + if let Some(token) = cancel_token { + token.cancel(); + } + if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { + tokio::spawn(async move { + ssm.teardown_preempted_stream(&stream_url, alloc_id).await; + allocation.release(); + }); + PreemptionOutcome::Exhausted + } else { + allocation.release(); + if let Some(alloc) = self.try_acquire_allocation_after_freed( + input_name, + allow_grace, + &reserved_providers, + session_owner, + ) { + PreemptionOutcome::Acquired(alloc) + } else { + PreemptionOutcome::Exhausted + } + } + } + PriorityOwner::Single(victim_alloc_id) => { + let (action, completion_token) = { + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get(&victim_alloc_id) { + if info.priority != v_prio || info.created_at != victim_created_at { + return PreemptionOutcome::Exhausted; + } + } else { + return PreemptionOutcome::Exhausted; + } + if let Some(info) = connections.single.get_mut(&victim_alloc_id) { + debug_if_enabled!( + "Preempting single connection from {} (prio={v_prio}) for higher priority request (prio={new_priority})", + sanitize_sensitive_info(&info.client_addr.to_string()) + ); + info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); + } + let completion_token = + connections.single.get(&victim_alloc_id).map(|info| info.completion_token.clone()); + let action = + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); + (action, completion_token) + }; + + match action { + ReleaseAction::Wait(victim_id, gen, comp_token) => { + let shutdown_token = self.shutdown_token.clone(); + let core = Arc::clone(&self.core); + let reaper_token = comp_token.clone(); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = reaper_token.cancelled() => { + core.complete_release_with_generation(victim_id, Some(gen)); + } + } + }); + PreemptionOutcome::PendingCompletion(comp_token) + } + ReleaseAction::None => { + if let Some(token) = completion_token { + PreemptionOutcome::PendingCompletion(token) + } else { + PreemptionOutcome::Exhausted + } + } + ReleaseAction::Immediate => { + if let Some(alloc) = self.try_acquire_allocation_after_freed( + input_name, + allow_grace, + &reserved_providers, + session_owner, + ) { + PreemptionOutcome::Acquired(alloc) + } else { + PreemptionOutcome::Exhausted + } + } + } + } + } + } + fn try_preempt_connection( &self, input_name: &Arc, @@ -1250,144 +1630,10 @@ impl ActiveProviderManager { kind_needed: ConnectionKind, session_owner: Option<&str>, ) -> Option { - let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); - let victim = { - let connections = self.read_connections(); - self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) - }; - - if let Some((owner, alloc_id, v_prio, victim_created_at)) = victim { - match owner { - PriorityOwner::Shared(shared_id) => { - debug_if_enabled!( - "Preempting shared connection (allocation_id={shared_id}, prio={v_prio}) for higher priority request (prio={new_priority})" - ); - let released_shared_allocation = { - let mut connections = self.write_connections(); - let key = connections.shared.shared_by_allocation_id.get(&shared_id).cloned()?; - - // Revalidate the selected shared victim under WRITE lock. - let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { - shared.allocation_id == alloc_id - && shared.priority == v_prio - && shared.created_at == victim_created_at - }); - if !still_match { - None - } else if let Some(shared) = connections.shared.by_key.remove(&key) { - connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); - Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); - for subscriber_id in shared.connections.keys() { - connections.shared.key_by_subscriber.remove(subscriber_id); - } - - if let Some(name) = shared.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&shared.allocation_id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(v_prio, Reverse(victim_created_at), alloc_id), - shared.kind, - ); - } - Some((key, shared.allocation, shared.cancel_token)) - } else { - None - } - }; - let (stream_url, allocation, cancel_token) = released_shared_allocation?; - if let Some(token) = cancel_token { - token.cancel(); - } - allocation.release(); - // Shared broadcast has to be torn down explicitly, otherwise the provider - // stream may continue after allocation counters were already released. - if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { - tokio::spawn(async move { - ssm.teardown_preempted_stream(&stream_url, alloc_id).await; - }); - } else { - error!( - "SharedStreamManager not initialised during preemption teardown for {}; \ - shared stream may linger after allocation release", - sanitize_sensitive_info(&stream_url) - ); - } - } - PriorityOwner::Single(victim_alloc_id) => { - let removed_info = { - let mut connections = self.write_connections(); - if let Some(info) = connections.single.get(&victim_alloc_id) { - if info.priority != v_prio || info.created_at != victim_created_at { - return None; - } - } - if let Some(info) = connections.single.remove(&victim_alloc_id) { - debug_if_enabled!( - "Preempting single connection from {} (prio={v_prio}) for higher priority request (prio={new_priority})", - sanitize_sensitive_info(&info.client_addr.to_string()) - ); - Self::unindex_owner(&mut connections, victim_alloc_id, info.session_owner.as_ref()); - if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { - set.remove(&victim_alloc_id); - if set.is_empty() { - connections.single_by_addr.remove(&info.client_addr); - } - } - if let Some(name) = info.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&victim_alloc_id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(v_prio, Reverse(victim_created_at), victim_alloc_id), - info.kind, - ); - } - Some(info) - } else { - None - } - }; - - let Some(info) = removed_info else { - // Another preemptor already removed this victim. - return None; - }; - - // Preempted probes must stop immediately; they must not keep a custom stream alive. - info.cancel_token.cancel(); - info.allocation.release(); - } - } - - // Now try acquire again preserving the original grace policy. - let attempts = self.providers.provider_names_for_input(input_name).len().max(1); - let mut excluded_providers = reserved_providers; - for _ in 0..attempts { - let allocation = self.providers.acquire_connection_with_grace_override_excluding( - input_name, - allow_grace, - &excluded_providers, - ); - if matches!(allocation, ProviderAllocation::Exhausted) { - break; - } - if let Some(provider_name) = allocation.get_provider_name() { - if self.exceeds_reserved_capacity(&provider_name, session_owner) { - excluded_providers.insert(provider_name); - allocation.release(); - continue; - } - } - return Some(allocation); - } + match self.try_preempt_connection_outcome(input_name, new_priority, allow_grace, kind_needed, session_owner) { + PreemptionOutcome::Acquired(alloc) => Some(alloc), + _ => None, } - - None } pub fn acquire_exact_connection_with_grace( @@ -1491,6 +1737,54 @@ impl ActiveProviderManager { self.acquire_connection_inner(input_name, allow_grace, &AcquireProviderParams { addr, priority, kind, lease }) } + /// Lineup acquisition with an explicit playback lease identity, awaiting preemption completion if needed. + pub async fn acquire_connection_with_lease_for_session_await( + &self, + input_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let params = AcquireProviderParams { addr, priority, kind, lease }; + let outcome = { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) { + return Some(handle); + } + self.try_preempt_connection_outcome( + input_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) + }; + + match outcome { + PreemptionOutcome::Acquired(alloc) => { + let _transition = self.lock_capacity_transition(); + Some(self.register_allocation(alloc, ¶ms)) + } + PreemptionOutcome::PendingCompletion(completion_token) => { + let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await; + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) + } + PreemptionOutcome::Exhausted => None, + } + } + /// Exact-provider acquisition with an explicit playback lease identity. pub fn acquire_exact_connection_with_lease_for_session( &self, @@ -1509,6 +1803,54 @@ impl ActiveProviderManager { ) } + /// Exact-provider acquisition with an explicit playback lease identity, awaiting preemption completion if needed. + pub async fn acquire_exact_connection_with_lease_for_session_await( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let params = AcquireProviderParams { addr, priority, kind, lease }; + let outcome = { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) { + return Some(handle); + } + self.try_preempt_connection_outcome( + provider_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) + }; + + match outcome { + PreemptionOutcome::Acquired(alloc) => { + let _transition = self.lock_capacity_transition(); + Some(self.register_allocation(alloc, ¶ms)) + } + PreemptionOutcome::PendingCompletion(completion_token) => { + let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await; + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) + } + PreemptionOutcome::Exhausted => None, + } + } + /// Acquire a provider connection for probe tasks with configurable priority. /// Probes never consume grace capacity. pub fn acquire_connection_for_probe(&self, input_name: &Arc, priority: i8) -> Option { @@ -1653,47 +1995,100 @@ impl ActiveProviderManager { } } - #[allow(clippy::too_many_lines)] + pub fn register_body_owner(&self, alloc_id: AllocationId) -> bool { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle == ConnectionLifecycle::Closing || info.lifecycle == ConnectionLifecycle::Closed { + return false; + } + info.has_body_owner = true; + if info.lifecycle == ConnectionLifecycle::Opening { + info.lifecycle = ConnectionLifecycle::Active; + } + true + } else { + false + } + } + + pub fn mark_opening(&self, alloc_id: AllocationId) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { + info.lifecycle = ConnectionLifecycle::Opening; + } + } + } + + pub fn renew_opening_tokens(&self, alloc_id: AllocationId) -> Option<(CancellationToken, CancellationToken)> { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { + info.lifecycle = ConnectionLifecycle::Opening; + info.open_generation = info.open_generation.wrapping_add(1); + info.reaper_spawned = false; + let cancel_token = CancellationToken::new(); + let completion_token = CancellationToken::new(); + info.cancel_token = cancel_token.clone(); + info.completion_token = completion_token.clone(); + return Some((cancel_token, completion_token)); + } + } + None + } + + pub fn mark_closing(&self, alloc_id: AllocationId) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + let prio = if let Some(info) = connections.single.get_mut(&alloc_id) { + info.lifecycle = ConnectionLifecycle::Closing; + info.cancel_token.cancel(); + info.allocation + .get_provider_name() + .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)) + } else { + None + }; + if let Some((name, key, kind)) = prio { + Self::remove_priority_entry(&mut connections, &name, &key, kind); + } + } + pub fn release_connection(&self, addr: &SocketAddr) { let _transition = self.lock_capacity_transition(); - // Single connection - all index updates in one lock scope - let single_allocations = { + let single_alloc_ids = { let mut connections = self.write_connections(); - if let Some(alloc_ids) = connections.single_by_addr.remove(addr) { - let mut allocations = Vec::with_capacity(alloc_ids.len()); - for id in alloc_ids { - if let Some(info) = connections.single.remove(&id) { - Self::unindex_owner(&mut connections, id, info.session_owner.as_ref()); - if !info.allocation.is_unlimited_provider() { - if let Some(name) = info.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(info.priority, Reverse(info.created_at), id), - info.kind, - ); - } - } - allocations.push(info); - } - } - Some(allocations) - } else { - None - } + connections.single_by_addr.remove(addr) }; - if let Some(allocations) = single_allocations { - for info in allocations { - debug_if_enabled!( - "Released provider connection {:?} for {}", - info.allocation.get_provider_name().unwrap_or_default(), - sanitize_sensitive_info(&addr.to_string()) - ); - info.allocation.release(); + if let Some(alloc_ids) = single_alloc_ids { + let mut to_reap = Vec::new(); + { + let mut connections = self.write_connections(); + for id in alloc_ids { + if let ReleaseAction::Wait(alloc_id, gen, token) = + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, id) + { + to_reap.push((alloc_id, gen, token)); + } + } + } + for (id, gen, completion_token) in to_reap { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { + rt_handle.spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(id, Some(gen)); + } + } + }); + } } } @@ -1705,77 +2100,72 @@ impl ActiveProviderManager { } } - pub fn release_handle(&self, handle: &ProviderHandle) { - let _transition = self.lock_capacity_transition(); + fn release_shared_handle_locked( + connections: &mut Connections, + handle: &ProviderHandle, + ) -> Option { let mut released = None; let mut released_priority_key: Option<(Arc, PriorityKey, ConnectionKind)> = None; - { - let mut connections = self.write_connections(); - - // Try removing from Single directly by allocation_id - if let Some(info) = connections.single.remove(&handle.allocation_id) { - Self::unindex_owner(&mut connections, handle.allocation_id, info.session_owner.as_ref()); - let pkey = (info.priority, Reverse(info.created_at), handle.allocation_id); - released = Some(info.allocation); - let released_kind = info.kind; - if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { - set.remove(&handle.allocation_id); - if set.is_empty() { - connections.single_by_addr.remove(&info.client_addr); - } + if let Some(key) = connections.shared.shared_by_allocation_id.remove(&handle.allocation_id) { + if let Some(shared) = connections.shared.by_key.remove(&key) { + Self::unindex_owner(connections, handle.allocation_id, shared.session_owner.as_ref()); + let pkey = (shared.priority, Reverse(shared.created_at), handle.allocation_id); + let shared_kind = shared.kind; + let shared_is_unlimited = shared.allocation.is_unlimited_provider(); + released = Some(shared.allocation); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); } - - // Remove from by_provider index. Skip for unlimited providers: - // they were never inserted by `register_allocation`, so the - // keys/values are not present and a removal would be a no-op - // at best and could race with concurrent unrelated operations - // on the same HashSet entry at worst. - if !released.as_ref().is_some_and(ProviderAllocation::is_unlimited_provider) { + if !shared_is_unlimited { if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) { if let Some(list) = connections.by_provider.get_mut(&name) { list.remove(&handle.allocation_id); } - released_priority_key = Some((name, pkey, released_kind)); + released_priority_key = Some((name, pkey, shared_kind)); } } } - - if released.is_none() { - // Try removing from Shared - if let Some(key) = connections.shared.shared_by_allocation_id.remove(&handle.allocation_id) { - if let Some(shared) = connections.shared.by_key.remove(&key) { - Self::unindex_owner(&mut connections, handle.allocation_id, shared.session_owner.as_ref()); - let pkey = (shared.priority, Reverse(shared.created_at), handle.allocation_id); - let shared_kind = shared.kind; - let shared_is_unlimited = shared.allocation.is_unlimited_provider(); - released = Some(shared.allocation); - for subscriber_id in shared.connections.keys() { - connections.shared.key_by_subscriber.remove(subscriber_id); - } - if !shared_is_unlimited { - if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&handle.allocation_id); - } - released_priority_key = Some((name, pkey, shared_kind)); - } - } - } - } - } - - // Remove from priority_index - if let Some((name, pkey, kind)) = &released_priority_key { - Self::remove_priority_entry(&mut connections, name, pkey, *kind); - } } - - // Releasing the physical provider allocation never concludes the playback - // request: the request end is applied separately via `finish_*` with the - // actual outcome, so a generic handle drop cannot invent a clean end. - if let Some(allocation) = released { + if let Some((name, pkey, kind)) = &released_priority_key { + Self::remove_priority_entry(connections, name, pkey, *kind); + } + if let Some(allocation) = released.as_ref() { allocation.release(); } + released + } + + pub fn release_handle(&self, handle: &ProviderHandle) { + let _transition = self.lock_capacity_transition(); + let wait_action = { + let mut connections = self.write_connections(); + if connections.single.contains_key(&handle.allocation_id) { + if handle.completion_token.as_ref().is_some_and(CancellationToken::is_cancelled) { + if let Some(info) = connections.single.get_mut(&handle.allocation_id) { + info.completion_token.cancel(); + } + } + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, handle.allocation_id) + } else { + Self::release_shared_handle_locked(&mut connections, handle); + ReleaseAction::None + } + }; + + if let ReleaseAction::Wait(alloc_id, gen, completion_token) = wait_action { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { + rt_handle.spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); + } + } + }); + } + } } /// Stops only stale requests of this playback, even on a multiplexed proxy socket. @@ -1795,6 +2185,8 @@ impl ActiveProviderManager { allocation_id: info.allocation_id, allocation: info.allocation.clone(), cancel_token: Some(info.cancel_token.clone()), + completion_token: Some(info.completion_token.clone()), + close_reason: Arc::clone(&info.close_reason), }); } } @@ -1804,6 +2196,7 @@ impl ActiveProviderManager { handles }; for handle in handles { + handle.set_close_reason(ProviderCloseReason::Superseded); if let Some(token) = &handle.cancel_token { token.cancel(); } @@ -1811,6 +2204,79 @@ impl ActiveProviderManager { } } + /// Stops stale requests of this playback and awaits upstream body closure before releasing capacity. + pub async fn release_playback_connections_await(&self, owner: &str, addrs: &[SocketAddr]) { + let targets = { + let mut connections = self.write_connections(); + let mut targets = Vec::new(); + for addr in addrs { + if let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() { + for id in alloc_ids { + if let Some(info) = connections.single.get_mut(&id) { + if info.session_owner.as_deref() == Some(owner) + && info.lifecycle != ConnectionLifecycle::Closed + { + info.lifecycle = ConnectionLifecycle::Closing; + info.close_reason.store(ProviderCloseReason::Superseded as u8, Ordering::Release); + targets.push(( + info.allocation_id, + info.cancel_token.clone(), + info.completion_token.clone(), + info.priority, + info.created_at, + info.kind, + info.allocation.get_provider_name(), + info.open_generation, + )); + info.reaper_spawned = true; + } + } + } + } + } + for (alloc_id, _, _, priority, created_at, kind, provider_name, _) in &targets { + if let Some(name) = provider_name { + Self::remove_priority_entry( + &mut connections, + name, + &(*priority, Reverse(*created_at), *alloc_id), + *kind, + ); + } + } + targets + }; + + if targets.is_empty() { + return; + } + + for (_, cancel_token, _, _, _, _, _, _) in &targets { + cancel_token.cancel(); + } + + let futures: Vec<_> = targets.iter().map(|(_, _, token, _, _, _, _, _)| token.cancelled()).collect(); + let _ = tokio::time::timeout(Duration::from_millis(1000), futures::future::join_all(futures)).await; + + for (alloc_id, _, completion_token, _, _, _, _, gen) in targets { + if completion_token.is_cancelled() { + self.core.complete_release_with_generation(alloc_id, Some(gen)); + } else { + log::warn!("Superseded provider connection {alloc_id} did not close within deadline, retaining slot as Closing until reaped"); + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); + } + } + }); + } + } + } + pub fn reclassify_connection(&self, addr: &SocketAddr, kind: ConnectionKind, priority: i8) -> bool { self.reclassify_connection_for_owner(addr, None, kind, priority) } @@ -3535,6 +4001,108 @@ mod tests { manager.release_handle(&alloc_a); } + #[tokio::test] + async fn test_preemption_respects_registered_body_owner_completion() { + let app_cfg = build_test_app_config(None, 1); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_1: SocketAddr = "127.0.0.1:49061".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:49062".parse().unwrap(); + + // 1. First request with prio 5 occupies limit=1 + let handle_1 = manager + .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) + .expect("first allocation should succeed"); + + // 2. Register body owner on handle_1 + manager.register_body_owner(handle_1.allocation_id); + + // 3. Higher-priority request with prio -1 arrives without grace while handle_1 is active + let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); + + // 4. Replacement must NOT be admitted before completion + assert!(handle_2.is_none(), "replacement must not be admitted before victim completion"); + assert!( + handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), + "victim should receive cancel signal" + ); + assert_eq!( + handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), + tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, + "victim close reason must be PriorityPreempted" + ); + + // 5. Victim completes upstream reading + handle_1.completion_token.as_ref().expect("completion token").cancel(); + + // Allow background complete_release task to run + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(20)).await; + + // 6. Now replacement request can be admitted + let handle_2 = manager + .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) + .expect("replacement should succeed after victim has completed release"); + + manager.release_handle(&handle_2); + } + + #[tokio::test] + async fn test_preemption_respects_opening_connection_completion() { + let app_cfg = build_test_app_config(None, 1); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_1: SocketAddr = "127.0.0.1:49063".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:49064".parse().unwrap(); + + // 1. First request with prio 5 occupies limit=1 + let handle_1 = manager + .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) + .expect("first allocation should succeed"); + manager.mark_opening(handle_1.allocation_id); + + // Verify lifecycle is Opening and body owner is not yet registered + { + let connections = manager.read_connections(); + let info = connections.single.get(&handle_1.allocation_id).expect("handle_1 exists"); + assert_eq!(info.lifecycle, tuliprox_core::model::ConnectionLifecycle::Opening); + assert!(!info.has_body_owner); + } + + // 2. Higher-priority request with prio -1 arrives without grace while handle_1 is in Opening + let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); + + // 3. Replacement must NOT be admitted before completion of opening + assert!(handle_2.is_none(), "replacement must not be admitted while victim is still opening"); + assert!( + handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), + "victim should receive cancel signal during open" + ); + assert_eq!( + handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), + tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, + "victim close reason must be PriorityPreempted" + ); + + // 4. Victim's open future completes and signals completion + handle_1.completion_token.as_ref().expect("completion token").cancel(); + + // Allow background complete_release task to run + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(20)).await; + + // 5. Now replacement request can be admitted + let handle_2 = manager + .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) + .expect("replacement should succeed after victim open future has completed release"); + + manager.release_handle(&handle_2); + } + #[tokio::test] async fn test_owner_index_consistent_through_lifecycle() { let app_cfg = build_test_app_config(None, 2); @@ -4166,4 +4734,159 @@ mod tests { let usage = manager.provider_lease_usage(&input_name); assert_eq!(usage.total(), 0, "lease table must return to baseline after churn"); } + + #[tokio::test] + async fn closing_state_blocks_slot_release_until_completion_token_cancelled() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let handle1 = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_001)), + false, + 0, + ConnectionKind::Normal, + Some("session-1"), + ) + .expect("first allocation should succeed"); + assert_eq!(manager.get_provider_connections_count(), 1); + + let completion_token = handle1.completion_token.clone().expect("completion token present"); + let alloc_id = handle1.allocation_id; + + // Mark Closing (simulating supersede / release_playback_connections_await) + manager.mark_closing(alloc_id); + + // Drop the handle while completion_token is still pending (upstream body not yet closed) + manager.release_handle(&handle1); + + // The slot must still be occupied by the Closing allocation + assert_eq!(manager.get_provider_connections_count(), 1); + + // A second start attempt must fail because the provider is at its limit (1 connection) + let handle2 = manager.acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_002)), + false, + 0, + ConnectionKind::Normal, + Some("session-2"), + ); + assert!(handle2.is_none(), "second start must be rejected while first slot is Closing"); + + // Now signal completion (upstream body owner dropped upstream socket) + completion_token.cancel(); + + // Allow background reaper task to execute complete_release + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!(manager.get_provider_connections_count(), 0); + + // Second start attempt now succeeds + let handle3 = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_003)), + false, + 0, + ConnectionKind::Normal, + Some("session-3"), + ) + .expect("start must succeed after completion releases the slot"); + assert_eq!(manager.get_provider_connections_count(), 1); + + manager.release_handle(&handle3); + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[tokio::test] + async fn reaper_cleanup_is_idempotent_and_removes_all_indices() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 47_001)), + false, + 0, + ConnectionKind::Normal, + Some("session-idempotent"), + ) + .expect("allocation should succeed"); + + let completion_token = handle.completion_token.clone().expect("completion token present"); + let alloc_id = handle.allocation_id; + manager.mark_closing(alloc_id); + + manager.release_handle(&handle); + completion_token.cancel(); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(manager.get_provider_connections_count(), 0); + + // Duplicate releases must be completely safe and no-op + manager.complete_release(alloc_id); + manager.release_handle(&handle); + manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 47_001))); + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[tokio::test] + async fn test_single_request_preemption_awaits_victim_completion() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let low_addr = SocketAddr::from(([127, 0, 0, 1], 48_001)); + let high_addr = SocketAddr::from(([127, 0, 0, 1], 48_002)); + + let low_handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &low_addr, + false, + 10, + ConnectionKind::Normal, + Some("session-victim"), + ) + .expect("initial low-priority allocation succeeds"); + + manager.mark_opening(low_handle.allocation_id); + assert!(manager.register_body_owner(low_handle.allocation_id)); + + let cancel_token = low_handle.cancel_token.clone().unwrap(); + let completion_token = low_handle.completion_token.clone().unwrap(); + + // Simulate upstream body owner: when cancelled, complete release after 30ms + let comp_clone = completion_token.clone(); + tokio::spawn(async move { + cancel_token.cancelled().await; + tokio::time::sleep(Duration::from_millis(30)).await; + comp_clone.cancel(); + }); + + // High priority acquire should preempt the victim, await its completion, and succeed in the same call + let high_handle = manager + .acquire_connection_with_lease_for_session_await( + &input_name, + &high_addr, + false, + 0, + ConnectionKind::Normal, + None, + ) + .await + .expect("high-priority allocation must succeed in the same call after awaiting victim completion"); + + assert_eq!(manager.get_provider_connections_count(), 1); + manager.release_handle(&high_handle); + assert_eq!(manager.get_provider_connections_count(), 0); + } } diff --git a/backend/session/src/active_user_manager/mod.rs b/backend/session/src/active_user_manager/mod.rs index c4bd58156..bffa4e726 100644 --- a/backend/session/src/active_user_manager/mod.rs +++ b/backend/session/src/active_user_manager/mod.rs @@ -1556,7 +1556,7 @@ impl ActiveUserManager { addr_count == 1 } }) - .map(|s| crate::EvictionCandidate { addr: s.addr, client_ip: s.client_ip.clone(), ts: s.ts }) + .map(|s| crate::EvictionCandidate { addr: s.addr, client_ip: s.client_ip.clone(), ts: s.ts, uid: s.uid }) .collect(); candidates } diff --git a/backend/session/src/admission_strategy.rs b/backend/session/src/admission_strategy.rs index 14dc96dd4..500556300 100644 --- a/backend/session/src/admission_strategy.rs +++ b/backend/session/src/admission_strategy.rs @@ -109,8 +109,8 @@ fn select_candidate<'a>( let should_replace = match selected { None => true, Some(current) => match order { - EvictionOrder::Oldest => candidate.ts < current.ts, - EvictionOrder::Latest => candidate.ts > current.ts, + EvictionOrder::Oldest => (candidate.ts, candidate.uid) < (current.ts, current.uid), + EvictionOrder::Latest => (candidate.ts, candidate.uid) > (current.ts, current.uid), }, }; if should_replace { @@ -125,6 +125,8 @@ pub struct EvictionCandidate { pub addr: SocketAddr, pub client_ip: String, pub ts: u64, + /// Monotonic stream identity used to order streams created in the same second. + pub uid: u32, } #[cfg(test)] @@ -135,7 +137,7 @@ mod tests { fn addr(port: u16) -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port) } fn candidate(port: u16, ip: &str, ts: u64) -> EvictionCandidate { - EvictionCandidate { addr: addr(port), client_ip: ip.to_string(), ts } + EvictionCandidate { addr: addr(port), client_ip: ip.to_string(), ts, uid: u32::from(port) } } #[test] @@ -183,6 +185,13 @@ mod tests { assert_evict(AdmissionStrategy::EvictUserLatest, &candidates, 1000); } + #[test] + fn equal_timestamp_uses_monotonic_stream_identity() { + let candidates = vec![candidate(1000, "1.1.1.1", 100), candidate(1001, "1.1.1.1", 100)]; + assert_evict(AdmissionStrategy::EvictUserSameIpOldest, &candidates, 1000); + assert_evict(AdmissionStrategy::EvictUserSameIpLatest, &candidates, 1001); + } + #[test] fn no_same_ip_candidate_returns_no_match() { let ctx = StrategyContext { username: "user1", client_ip: "1.1.1.1" }; diff --git a/backend/session/src/stream.rs b/backend/session/src/stream.rs index 11a5fcbde..ce1c04fee 100644 --- a/backend/session/src/stream.rs +++ b/backend/session/src/stream.rs @@ -21,6 +21,7 @@ pub struct ProviderStreamFactoryResponse { pub stream: BoxedProviderStream, pub info: ProviderStreamInfo, pub provider_session_headers: HashMap, + pub has_upstream_owner: bool, } /// Controls whether a provider stream preserves its origin representation or normalizes it to identity bytes. @@ -85,6 +86,7 @@ pub struct StreamDetails { /// Set when the stream was admitted via a user-grace strategy. Carried through to /// `stream_grace_period` so remaining strategies can be evaluated if the grace fails. pub grace_resolution_context: Option, + pub custom_reason: Option, } /// Manual Clone: stream and `provider_handle` cannot be duplicated so we set them to None on the clone. @@ -107,6 +109,7 @@ impl Clone for StreamDetails { provider_handle: None, content_representation: self.content_representation, grace_resolution_context: self.grace_resolution_context.clone(), + custom_reason: self.custom_reason, } } } @@ -129,6 +132,7 @@ impl StreamDetails { provider_handle: None, content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, } } #[inline] diff --git a/backend/session/src/streams/mod.rs b/backend/session/src/streams/mod.rs index ffc980b7d..c48d1b2a0 100644 --- a/backend/session/src/streams/mod.rs +++ b/backend/session/src/streams/mod.rs @@ -1,4 +1,5 @@ pub mod buffered_stream; +pub mod provider_body_owner; pub mod shared_stream_manager; -pub use self::{buffered_stream::*, shared_stream_manager::*}; +pub use self::{buffered_stream::*, provider_body_owner::*, shared_stream_manager::*}; diff --git a/backend/session/src/streams/provider_body_owner.rs b/backend/session/src/streams/provider_body_owner.rs new file mode 100644 index 000000000..5ea6d952b --- /dev/null +++ b/backend/session/src/streams/provider_body_owner.rs @@ -0,0 +1,375 @@ +use crate::BoxedProviderStream; +use bytes::Bytes; +use futures::{ + stream::Stream, + task::{Context, Poll}, + StreamExt, +}; +use log::debug; +use std::{ + cmp::max, + future::Future, + pin::Pin, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + select, + sync::{ + mpsc::{channel, error::TrySendError, Sender}, + Semaphore, + }, + time::{sleep, Instant}, +}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +use tuliprox_core::model::{ProviderCloseReason, StreamError}; + +pub const DEFAULT_DIRECT_BODY_CHANNEL_SIZE: usize = 2; +pub const DEFAULT_DIRECT_BODY_MAX_BYTES: usize = 256 * 1024; +pub const DEFAULT_DIRECT_BODY_IDLE_SECS: u64 = 90; + +pub const DEFAULT_BUFFERED_CHANNEL_SIZE: usize = 1024; +pub const DEFAULT_BUFFERED_MAX_BYTES: usize = 5 * 1024 * 1024; +pub const DEFAULT_BUFFERED_IDLE_SECS: u64 = 10; + +#[derive(Debug, Clone, Copy)] +pub struct ProviderBodyOwnerConfig { + pub channel_capacity: usize, + pub max_buffer_bytes: usize, + pub idle_timeout: Duration, +} + +impl ProviderBodyOwnerConfig { + pub fn for_direct_body() -> Self { + Self { + channel_capacity: DEFAULT_DIRECT_BODY_CHANNEL_SIZE, + max_buffer_bytes: DEFAULT_DIRECT_BODY_MAX_BYTES, + idle_timeout: Duration::from_secs(DEFAULT_DIRECT_BODY_IDLE_SECS), + } + } + + pub fn for_buffered(buffer_size: usize, max_buffer_bytes: usize) -> Self { + Self { + channel_capacity: max(buffer_size, DEFAULT_BUFFERED_CHANNEL_SIZE), + max_buffer_bytes: if max_buffer_bytes == 0 { DEFAULT_BUFFERED_MAX_BYTES } else { max_buffer_bytes }, + idle_timeout: Duration::from_secs(DEFAULT_BUFFERED_IDLE_SECS), + } + } +} + +/// An upstream body owner task that decouples upstream body consumption from downstream polling. +/// It observes cancellation independently of downstream `poll_next`, ensuring that upstream +/// connections close promptly upon preemption or client disconnect. +pub struct ProviderBodyOwner { + stream: ReceiverStream>, + close_cancelled: Pin>, + cancel_token: CancellationToken, + semaphore: Arc, + max_buffer_bytes: usize, +} + +impl ProviderBodyOwner { + pub fn new( + stream: BoxedProviderStream, + config: ProviderBodyOwnerConfig, + cancel_token: CancellationToken, + completion_token: Option, + close_reason: Option>, + ) -> Self { + let (tx, rx) = channel(config.channel_capacity); + let semaphore = Arc::new(Semaphore::new(config.max_buffer_bytes)); + + tokio::spawn(Self::run_owner_task( + tx, + stream, + cancel_token.clone(), + completion_token, + close_reason, + Arc::clone(&semaphore), + config, + )); + + Self { + stream: ReceiverStream::new(rx), + close_cancelled: Box::pin(cancel_token.clone().cancelled_owned()), + cancel_token, + semaphore, + max_buffer_bytes: config.max_buffer_bytes, + } + } + + async fn run_owner_task( + tx: Sender>, + mut stream: BoxedProviderStream, + cancel_token: CancellationToken, + completion_token: Option, + close_reason: Option>, + semaphore: Arc, + config: ProviderBodyOwnerConfig, + ) { + let idle_timeout = config.idle_timeout; + let idle = sleep(idle_timeout); + tokio::pin!(idle); + + while !cancel_token.is_cancelled() { + select! { + biased; + () = cancel_token.cancelled() => { + debug!("Provider body owner task observed cancellation signal"); + break; + } + () = &mut idle => { + debug!("Provider body owner task idle timeout expired ({idle_timeout:?})"); + if let Some(reason) = &close_reason { + let _ = reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + ProviderCloseReason::IdleTimeout as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + cancel_token.cancel(); + break; + } + chunk = stream.next() => { + idle.as_mut().reset(Instant::now() + idle_timeout); + match chunk { + Some(Ok(bytes)) => { + let chunk_len = bytes.len(); + let permits = chunk_len.min(config.max_buffer_bytes); + if permits > 0 { + let acquired = select! { + biased; + () = cancel_token.cancelled() => None, + permit = Arc::clone(&semaphore).acquire_many_owned(u32::try_from(permits).unwrap_or(u32::MAX)) => permit.ok(), + }; + let Some(permit) = acquired else { + cancel_token.cancel(); + break; + }; + permit.forget(); + } + let send_res = match tx.try_send(Ok(bytes)) { + Ok(()) => Ok(()), + Err(TrySendError::Full(item)) => { + select! { + biased; + () = cancel_token.cancelled() => Err(()), + res = tx.send(item) => res.map_err(|_| ()), + } + } + Err(TrySendError::Closed(_)) => Err(()), + }; + if send_res.is_err() { + if permits > 0 { + semaphore.add_permits(permits); + } + if let Some(reason) = &close_reason { + let _ = reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + ProviderCloseReason::ClientClosed as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + cancel_token.cancel(); + break; + } + } + Some(Err(err)) => { + if let Some(reason) = &close_reason { + let _ = reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + ProviderCloseReason::ProviderError as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + select! { + biased; + () = cancel_token.cancelled() => {}, + _ = tx.send(Err(err)) => {}, + } + break; + } + None => { + debug!("Provider body owner task completed cleanly on EOF"); + break; + } + } + } + } + } + + // Explicitly drop stream before signalling completion, releasing reqwest/hyper socket + drop(stream); + drop(tx); + + if let Some(completion) = completion_token { + completion.cancel(); + } + } +} + +impl Stream for ProviderBodyOwner { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + match Pin::new(&mut this.stream).poll_next(cx) { + Poll::Ready(Some(Ok(bytes))) => { + this.semaphore.add_permits(bytes.len().min(this.max_buffer_bytes)); + Poll::Ready(Some(Ok(bytes))) + } + Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => { + if this.close_cancelled.as_mut().poll(cx).is_ready() { + Poll::Ready(None) + } else { + Poll::Pending + } + } + } + } +} + +impl Drop for ProviderBodyOwner { + fn drop(&mut self) { self.cancel_token.cancel(); } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::Stream; + use tokio::sync::oneshot; + + struct GatedDropProbeStream { + gate: oneshot::Receiver<()>, + dropped: Option>, + yielded: bool, + } + + impl Stream for GatedDropProbeStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.yielded { + return Poll::Pending; + } + match Pin::new(&mut self.gate).poll(cx) { + Poll::Ready(Ok(())) => { + self.yielded = true; + Poll::Ready(Some(Ok(Bytes::from_static(b"chunk")))) + } + Poll::Ready(Err(_)) => Poll::Ready(None), + Poll::Pending => Poll::Pending, + } + } + } + + impl Drop for GatedDropProbeStream { + fn drop(&mut self) { + if let Some(dropped) = self.dropped.take() { + let _ = dropped.send(()); + } + } + } + + #[tokio::test] + async fn unpolled_body_owner_terminates_upstream_when_cancelled() { + let (gate_tx, gate_rx) = oneshot::channel(); + let (dropped_tx, dropped_rx) = oneshot::channel(); + let cancel = CancellationToken::new(); + let completion = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(0)); + + let upstream = GatedDropProbeStream { gate: gate_rx, dropped: Some(dropped_tx), yielded: false }; + let owner = ProviderBodyOwner::new( + Box::pin(upstream), + ProviderBodyOwnerConfig::for_direct_body(), + cancel.clone(), + Some(completion.clone()), + Some(close_reason.clone()), + ); + + // Cancel the owner WITHOUT EVER POLLING IT + cancel.cancel(); + + // Completion must fire + tokio::time::timeout(Duration::from_secs(1), completion.cancelled()) + .await + .expect("completion token must be signalled when owner task terminates"); + + // Upstream stream must have been dropped + gate_tx.send(()).unwrap_err(); // gate_rx is dropped when upstream is dropped! + + tokio::time::timeout(Duration::from_secs(1), dropped_rx) + .await + .expect("upstream stream must be dropped without downstream polling") + .expect("drop notification delivered"); + + drop(owner); + } + + #[tokio::test] + async fn error_delivery_does_not_block_cancellation_when_channel_full() { + let (dropped_tx, dropped_rx) = oneshot::channel(); + let cancel = CancellationToken::new(); + let completion = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(0)); + + struct TwoChunksThenErrStream { + count: usize, + dropped: Option>, + } + impl Stream for TwoChunksThenErrStream { + type Item = Result; + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.count < 2 { + self.count += 1; + Poll::Ready(Some(Ok(Bytes::from_static(b"chunk")))) + } else if self.count == 2 { + self.count += 1; + Poll::Ready(Some(Err(StreamError::Stream("test error".to_owned())))) + } else { + Poll::Pending + } + } + } + impl Drop for TwoChunksThenErrStream { + fn drop(&mut self) { + if let Some(d) = self.dropped.take() { + let _ = d.send(()); + } + } + } + + let upstream = TwoChunksThenErrStream { count: 0, dropped: Some(dropped_tx) }; + let owner = ProviderBodyOwner::new( + Box::pin(upstream), + ProviderBodyOwnerConfig::for_direct_body(), + cancel.clone(), + Some(completion.clone()), + Some(close_reason.clone()), + ); + + tokio::time::sleep(Duration::from_millis(50)).await; + cancel.cancel(); + + tokio::time::timeout(Duration::from_millis(500), completion.cancelled()) + .await + .expect("completion token must be signalled when owner task terminates on error delivery"); + + tokio::time::timeout(Duration::from_millis(500), dropped_rx) + .await + .expect("upstream stream must be dropped") + .expect("drop notification delivered"); + + drop(owner); + } +} diff --git a/bin/run-testkit-scenarios.sh b/bin/run-testkit-scenarios.sh new file mode 100755 index 000000000..619cd104f --- /dev/null +++ b/bin/run-testkit-scenarios.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd -P)" +cd "${REPO_ROOT}" + +TARGET_DIR="${CARGO_TARGET_DIR:-${REPO_ROOT}/target}" +SUT_BINARY="${TARGET_DIR}/debug/tuliprox" +TESTKIT_BINARY="${TARGET_DIR}/debug/tuliprox-testkit" +REPORT_DIR="${REPO_ROOT}/testkit-report" + +if [[ ! -x "${SUT_BINARY}" ]]; then + echo "❌ Tuliprox SUT binary not found at ${SUT_BINARY}. Build it first: cargo build -p tuliprox" >&2 + exit 1 +fi + +if [[ ! -x "${TESTKIT_BINARY}" ]]; then + echo "❌ Testkit binary not found at ${TESTKIT_BINARY}. Build it first: cargo build -p tuliprox-testkit" >&2 + exit 1 +fi + +mkdir -p "${REPORT_DIR}" +export TULIPROX_TESTKIT_SUT_BINARY="${SUT_BINARY}" + +SUITE_RUN_ID="${TULIPROX_TESTKIT_RUN_ID:-$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || echo "suite-$(date +%s)")}" + +FAILED_SCENARIOS=() +PASSED_COUNT=0 + +echo "🚀 Starting testkit scenario suite (run_id: ${SUITE_RUN_ID})..." + +for scenario_path in test/fixtures/testkit/scenarios/*.yml; do + scenario_name="$(basename "${scenario_path}" .yml)" + scenario_dir="${REPORT_DIR}/${scenario_name}" + rm -rf "${scenario_dir}" + mkdir -p "${scenario_dir}" + echo " ▶ Running scenario: ${scenario_name}" + + set +e + "${TESTKIT_BINARY}" controller --scenario "${scenario_path}" --report-directory "${scenario_dir}" --run-id "${SUITE_RUN_ID}" > "${scenario_dir}/scenario.log" 2>&1 + exit_code=$? + set -e + + if [[ ${exit_code} -eq 0 ]]; then + echo " ✅ ${scenario_name} PASSED" + PASSED_COUNT=$((PASSED_COUNT + 1)) + else + echo " ❌ ${scenario_name} FAILED (exit code ${exit_code})" + FAILED_SCENARIOS+=("${scenario_name}") + if [[ ! -f "${scenario_dir}/run.json" ]]; then + cat < "${scenario_dir}/run.json" +{ + "schema_version": 1, + "scenario": "${scenario_name}", + "outcome": "failed", + "policy_hash": "pre-start", + "run_id": "${SUITE_RUN_ID}", + "error_kind": "controller_crashed", + "error_message": "Controller exited with code ${exit_code} before run.json could be generated", + "exit_code": ${exit_code}, + "events": [] +} +EOF + cat < "${scenario_dir}/summary.txt" +scenario: ${scenario_name} +outcome: failed +policy: pre-start +run_id: ${SUITE_RUN_ID} +events: 0 +EOF + touch "${scenario_dir}/events.ndjson" + cat < "${scenario_dir}/junit.xml" + + +EOF + fi + cat "${scenario_dir}/scenario.log" >&2 + fi +done + +echo "" +echo "========================================" +echo "Testkit Scenario Summary:" +echo " Run ID: ${SUITE_RUN_ID}" +echo " Passed: ${PASSED_COUNT}" +echo " Failed: ${#FAILED_SCENARIOS[@]}" +echo "========================================" + +FAILED_JSON_ARRAY="[]" +if [[ ${#FAILED_SCENARIOS[@]} -gt 0 ]]; then + FAILED_JSON_ARRAY="[$(printf '"%s",' "${FAILED_SCENARIOS[@]}" | sed 's/,$//')]" +fi + +cat < "${REPORT_DIR}/summary.json" +{ + "run_id": "${SUITE_RUN_ID}", + "total": $((PASSED_COUNT + ${#FAILED_SCENARIOS[@]})), + "passed": ${PASSED_COUNT}, + "failed": ${#FAILED_SCENARIOS[@]}, + "failing_scenarios": ${FAILED_JSON_ARRAY} +} +EOF + +if [[ ${#FAILED_SCENARIOS[@]} -gt 0 ]]; then + echo "❌ Failing scenarios: ${FAILED_SCENARIOS[*]}" >&2 + exit 1 +fi + +echo "✅ All scenarios passed successfully!" diff --git a/docker/Dockerfile b/docker/Dockerfile index 65d7d9455..c76e9d1a6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,15 +35,19 @@ COPY backend/parser/Cargo.toml ./backend/parser/ COPY backend/processing/Cargo.toml ./backend/processing/ COPY backend/repository/Cargo.toml ./backend/repository/ COPY backend/session/Cargo.toml ./backend/session/ +COPY tools/tuliprox-testkit/Cargo.toml ./tools/tuliprox-testkit/ -RUN mkdir -p frontend/src backend/app/src/tools && \ +RUN mkdir -p frontend/src backend/app/src/tools tools/tuliprox-testkit/src && \ printf 'fn main() {}\n' > frontend/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/tools/flags_builder.rs && \ + printf '\n' > tools/tuliprox-testkit/src/lib.rs && \ + printf 'fn main() {}\n' > tools/tuliprox-testkit/src/main.rs && \ for crate in auth btree config-loader core curation dvr hls iptv library media-server metadata messaging mpegts parser processing repository session; do \ mkdir -p "backend/${crate}/src"; \ printf '\n' > "backend/${crate}/src/lib.rs"; \ - done + done && \ + cargo metadata --no-deps --format-version 1 > /dev/null # ----------------------------------------------------------------- # Stage 1: Build the Rust binary for production diff --git a/docker/cross.Dockerfile b/docker/cross.Dockerfile index 67c82fc8e..56d099caa 100644 --- a/docker/cross.Dockerfile +++ b/docker/cross.Dockerfile @@ -39,15 +39,19 @@ COPY backend/parser/Cargo.toml ./backend/parser/ COPY backend/processing/Cargo.toml ./backend/processing/ COPY backend/repository/Cargo.toml ./backend/repository/ COPY backend/session/Cargo.toml ./backend/session/ +COPY tools/tuliprox-testkit/Cargo.toml ./tools/tuliprox-testkit/ -RUN mkdir -p frontend/src backend/app/src/tools && \ +RUN mkdir -p frontend/src backend/app/src/tools tools/tuliprox-testkit/src && \ printf 'fn main() {}\n' > frontend/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/tools/flags_builder.rs && \ + printf '\n' > tools/tuliprox-testkit/src/lib.rs && \ + printf 'fn main() {}\n' > tools/tuliprox-testkit/src/main.rs && \ for crate in auth btree config-loader core curation dvr hls iptv library media-server metadata messaging mpegts parser processing repository session; do \ mkdir -p "backend/${crate}/src"; \ printf '\n' > "backend/${crate}/src/lib.rs"; \ - done + done && \ + cargo metadata --no-deps --format-version 1 > /dev/null # ----------------------------------------------------------------- # Stage 1: Build the Rust binary for production diff --git a/test/fixtures/testkit/custom-video-fixtures.json b/test/fixtures/testkit/custom-video-fixtures.json new file mode 100644 index 000000000..6098ae2bd --- /dev/null +++ b/test/fixtures/testkit/custom-video-fixtures.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "fixtures": { + "hls/channel_unavailable.ts": "afefcdbcf3749e4e7a89321239dfcb7d2c0bdc4d2d2fb1cf4c883962c837c92f", + "hls/hls_session_or_lease_expired.ts": "9fe61499c2080f4aa1e362dfb72ec7c0a4b283f08d7dcac1f21590bd4db2fe55", + "hls/low_priority_preempted.ts": "9cc3a7b63f02e8e0bc45fe3b59c18f52443ded43fa0c9121fb0ea676def412a9", + "hls/panel_api_provisioning.ts": "b481b7b85f79883b41f8a1436301257ee5cb5a45e30993233ec73a2a5824ff13", + "hls/provider_connections_exhausted.ts": "099181ddcf9e7edc5007db6d801348afb132e27382eb4e0d33589dbe07cee790", + "hls/user_account_expired.ts": "110ea30bdec0f4d6457dcbfd2c8fb8dc0b2f517d8dedb473de1191d6de369464", + "hls/user_connections_exhausted.ts": "6ab07858d03e19b68eea8cbaecb67ad4d276633c8b423253b7f39c585cab73fe" + } +} diff --git a/test/fixtures/testkit/scenarios/evict-same-ip-latest.yml b/test/fixtures/testkit/scenarios/evict-same-ip-latest.yml new file mode 100644 index 000000000..cd542b4a0 --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-same-ip-latest.yml @@ -0,0 +1,62 @@ +schema_version: 1 +name: evict-same-ip-latest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: evict-same-ip-latest +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-b + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-c + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-c + client_ip: { mode: forwarded, value: 10.20.0.10 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-a + start: + actor: tv-a + playback_id: playback-a + session_group: playback-a + channel: live-17 + await: + valid_frames: 5 + - command_id: start-b + start: + actor: tv-b + playback_id: playback-b + session_group: playback-b + channel: live-18 + await: + valid_frames: 5 + - command_id: start-c + expect: streaming + start: + actor: tv-c + playback_id: playback-c + session_group: playback-c + channel: live-19 +policy_contract: + user_access_control: true + users: + admission-user: + max_connections: 2 + admission_strategies: + - evict_user_same_ip_latest diff --git a/test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml b/test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml new file mode 100644 index 000000000..6131b4494 --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml @@ -0,0 +1,26 @@ +schema_version: 1 +name: evict-same-ip-oldest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: evict-same-ip-oldest } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-b, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-c, client_ip: { mode: forwarded, value: 10.20.0.10 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } } + - { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } } + - { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 2 } } + admission_strategies: [evict_user_same_ip_oldest] diff --git a/test/fixtures/testkit/scenarios/evict-user-latest.yml b/test/fixtures/testkit/scenarios/evict-user-latest.yml new file mode 100644 index 000000000..a5569c2b7 --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-user-latest.yml @@ -0,0 +1,26 @@ +schema_version: 1 +name: evict-user-latest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: evict-user-latest } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-b, client_ip: { mode: forwarded, value: 10.20.0.11 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-c, client_ip: { mode: forwarded, value: 10.20.0.12 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } } + - { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } } + - { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 2 } } + admission_strategies: [evict_user_latest] diff --git a/test/fixtures/testkit/scenarios/evict-user-oldest.yml b/test/fixtures/testkit/scenarios/evict-user-oldest.yml new file mode 100644 index 000000000..3915384df --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-user-oldest.yml @@ -0,0 +1,26 @@ +schema_version: 1 +name: evict-user-oldest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: evict-user-oldest } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-b, client_ip: { mode: forwarded, value: 10.20.0.11 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-c, client_ip: { mode: forwarded, value: 10.20.0.12 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } } + - { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } } + - { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 2 } } + admission_strategies: [evict_user_oldest] diff --git a/test/fixtures/testkit/scenarios/hard-limit.yml b/test/fixtures/testkit/scenarios/hard-limit.yml new file mode 100644 index 000000000..4c7e98b14 --- /dev/null +++ b/test/fixtures/testkit/scenarios/hard-limit.yml @@ -0,0 +1,47 @@ +schema_version: 1 +name: hard-limit-denies-second-playback +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: hard-limit-denies-second-playback +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-b + client_ip: { mode: forwarded, value: 10.20.0.10 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } +steps: + - command_id: start-a + start: + actor: tv-a + playback_id: playback-a + session_group: playback-a + channel: live-17 + await: + valid_frames: 5 + - command_id: start-b + expect: rejected + start: + actor: tv-b + playback_id: playback-b + session_group: playback-b + channel: live-17 +policy_contract: + user_access_control: true + users: + admission-user: + max_connections: 1 + soft_connections: 0 + admission_strategies: [] diff --git a/test/fixtures/testkit/scenarios/provider-hard-limit.yml b/test/fixtures/testkit/scenarios/provider-hard-limit.yml new file mode 100644 index 000000000..688bbe923 --- /dev/null +++ b/test/fixtures/testkit/scenarios/provider-hard-limit.yml @@ -0,0 +1,38 @@ +schema_version: 1 +name: provider-hard-limit-denies-distinct-stream +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: provider-hard-limit-denies-distinct-stream +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/provider-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/provider-b + client_ip: { mode: forwarded, value: 10.20.0.11 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } +steps: + - command_id: start-a + start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 } + await: { valid_frames: 5 } + - command_id: start-b + expect: rejected + start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 } +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 2 } + admission_strategies: [] + provider_max_connections: 1 diff --git a/test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml b/test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml new file mode 100644 index 000000000..86e09b23d --- /dev/null +++ b/test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml @@ -0,0 +1,38 @@ +schema_version: 1 +name: shared-stream-uses-one-provider-slot +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: shared-stream-uses-one-provider-slot +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/shared-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/shared-b + client_ip: { mode: forwarded, value: 10.20.0.11 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } +steps: + - command_id: start-a + start: { actor: tv-a, playback_id: playback-a, session_group: shared-live-17, channel: live-17 } + await: { valid_frames: 5 } + - command_id: start-b + start: { actor: tv-b, playback_id: playback-b, session_group: shared-live-17, channel: live-17 } + await: { valid_frames: 5 } +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 2 } + admission_strategies: [] + provider_max_connections: 1 + expected_provider_slots: 1 diff --git a/test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml b/test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml new file mode 100644 index 000000000..fa2824b90 --- /dev/null +++ b/test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml @@ -0,0 +1,99 @@ +schema_version: 1 +name: vod-range-reopen-preserves-three-live +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: vod-range-reopen-preserves-three-live + account_limit: 5 + limit_mode: reject_new +actors: + - id: live-viewer-1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-1 + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: live-viewer-2 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-2 + client_ip: { mode: forwarded, value: 10.20.0.11 } + - id: live-viewer-3 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-3 + client_ip: { mode: forwarded, value: 10.20.0.12 } + - id: vod-viewer-ua1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/vod-player-ua1 + client_ip: { mode: forwarded, value: 10.20.0.13 } + - id: vod-viewer-ua2 + agent: local + username: admission-user + user_agent: tuliprox-testkit/vod-player-ua2 + client_ip: { mode: forwarded, value: 10.20.0.13 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-live-1 + start: { actor: live-viewer-1, playback_id: live-1, session_group: live-1, channel: live-17 } + await: { valid_frames: 3 } + - command_id: start-live-2 + start: { actor: live-viewer-2, playback_id: live-2, session_group: live-2, channel: live-18 } + await: { valid_frames: 3 } + - command_id: start-live-3 + start: { actor: live-viewer-3, playback_id: live-3, session_group: live-3, channel: live-19 } + await: { valid_frames: 3 } + assert_origin: + active_body: 3 + max_active_body: 3 + max_active_tcp: 3 + - command_id: start-vod-probe-1 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-1 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=0-524287" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + - command_id: start-vod-probe-2 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-2 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=1048576-1572863" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + no_limit_rejections: true + no_evictions: true + - command_id: stop-vod-probe-2 + stop: { playback_id: vod-probe-2 } + - command_id: stop-vod-probe-1 + stop: { playback_id: vod-probe-1 } + assert_origin: + active_body: 3 + max_active_body: 4 + max_active_tcp: 4 +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 10 } + admission_strategies: [] + provider_max_connections: 4 diff --git a/test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml b/test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml new file mode 100644 index 000000000..53167db15 --- /dev/null +++ b/test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml @@ -0,0 +1,94 @@ +schema_version: 1 +name: vod-range-reopen-strict-cap +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: vod-range-reopen-strict-cap + account_limit: 4 + limit_mode: reject_new +actors: + - id: live-viewer-1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-1 + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: live-viewer-2 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-2 + client_ip: { mode: forwarded, value: 10.20.0.11 } + - id: live-viewer-3 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-3 + client_ip: { mode: forwarded, value: 10.20.0.12 } + - id: vod-viewer-ua1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/vod-player-ua1 + client_ip: { mode: forwarded, value: 10.20.0.13 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-live-1 + start: { actor: live-viewer-1, playback_id: live-1, session_group: live-1, channel: live-17 } + await: { valid_frames: 3 } + - command_id: start-live-2 + start: { actor: live-viewer-2, playback_id: live-2, session_group: live-2, channel: live-18 } + await: { valid_frames: 3 } + - command_id: start-live-3 + start: { actor: live-viewer-3, playback_id: live-3, session_group: live-3, channel: live-19 } + await: { valid_frames: 3 } + assert_origin: + active_body: 3 + max_active_body: 3 + max_active_tcp: 3 + - command_id: start-vod-probe-1 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-1 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=0-524287" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + - command_id: start-vod-probe-2 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-2 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=1048576-1572863" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + no_limit_rejections: true + no_evictions: true + - command_id: stop-vod-probe-2 + stop: { playback_id: vod-probe-2 } + - command_id: stop-vod-probe-1 + stop: { playback_id: vod-probe-1 } + assert_origin: + active_body: 3 + max_active_body: 4 + max_active_tcp: 4 +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 10 } + admission_strategies: [] + provider_max_connections: 4 diff --git a/test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml b/test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml new file mode 100644 index 000000000..aa73f4843 --- /dev/null +++ b/test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml @@ -0,0 +1,65 @@ +schema_version: 1 +name: vod-reopen-backpressured-body +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: vod-reopen-backpressured-body + account_limit: 2 + limit_mode: reject_new +actors: + - id: vod-viewer + agent: local + username: admission-user + user_agent: tuliprox-testkit/backpressured-vod + client_ip: { mode: forwarded, value: 10.20.0.20 } +channels: {} +steps: + # First probe: read only 16 KiB out of 512 KiB, then pause to create downstream backpressure + - command_id: start-vod-backpressured + start: + actor: vod-viewer + playback_id: vod-bp-1 + session_group: vod-bp + vod_object: movie.mkv + range: "bytes=0-524287" + read_limit_bytes: 16384 + post_read_action: pause + assert_origin: + active_body: 1 + max_active_body: 1 + max_active_tcp: 1 + # Second probe on the same session while first probe is still open and backpressured. + # Forced reopen must reclaim the single provider slot (max_connections: 1) without peak violations. + - command_id: start-vod-next + start: + actor: vod-viewer + playback_id: vod-bp-2 + session_group: vod-bp + vod_object: movie.mkv + range: "bytes=1048576-1572863" + read_limit_bytes: 65536 + post_read_action: pause + assert_origin: + active_body: 1 + max_active_body: 1 + max_active_tcp: 1 + no_limit_rejections: true + - command_id: stop-vod-next + stop: { playback_id: vod-bp-2 } + - command_id: stop-vod-backpressured + stop: { playback_id: vod-bp-1 } + assert_origin: + active_body: 0 + max_active_body: 1 + max_active_tcp: 1 +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 2 } + admission_strategies: [] + provider_max_connections: 1 diff --git a/tools/tuliprox-testkit/Cargo.toml b/tools/tuliprox-testkit/Cargo.toml new file mode 100644 index 000000000..978f20207 --- /dev/null +++ b/tools/tuliprox-testkit/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "tuliprox-testkit" +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +axum = { version = "0.8.9", features = ["macros", "default", "ws"] } +base64.workspace = true +blake3.workspace = true +bytes.workspace = true +clap = { version = "4.6.6", features = ["derive"] } +crc32fast = "1.5.1" +futures.workspace = true +reqwest = { version = "0.13.4", features = ["json", "stream", "rustls"] } +serde.workspace = true +serde_json.workspace = true +serde-saphyr.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "process", "signal", "sync", "time"] } +tokio-stream = "0.1.19" +tokio-tungstenite = "0.29.0" +tempfile = "3.27.0" +pin-project-lite = "0.2.17" +url.workspace = true +hyper = { version = "1.11.0", features = ["server", "http1"] } +hyper-util = { version = "0.1.20", features = ["server", "server-auto", "tokio"] } +tower = "0.5.3" +tower-service = "0.3.3" + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/tools/tuliprox-testkit/src/adaptive.rs b/tools/tuliprox-testkit/src/adaptive.rs new file mode 100644 index 000000000..922866ff6 --- /dev/null +++ b/tools/tuliprox-testkit/src/adaptive.rs @@ -0,0 +1,80 @@ +use crate::TestkitError; +use std::{collections::HashMap, time::Duration}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdaptiveSessionState { + Prepared, + Active, + Preserved, +} + +#[derive(Debug)] +struct Session { + state: AdaptiveSessionState, + expires_at: tokio::time::Instant, +} + +#[derive(Debug, Default)] +pub struct AdaptiveSessionTracker { + sessions: HashMap, +} + +impl AdaptiveSessionTracker { + pub fn manifest(&mut self, session_group: String, ttl: Duration) -> Result { + if self.sessions.contains_key(&session_group) { + return Err(TestkitError::Protocol("adaptive manifest created a duplicate logical session".to_owned())); + } + self.sessions.insert( + session_group, + Session { state: AdaptiveSessionState::Prepared, expires_at: tokio::time::Instant::now() + ttl }, + ); + Ok(AdaptiveSessionState::Prepared) + } + + pub fn segment(&mut self, session_group: &str, ttl: Duration) -> Result { + let session = self.sessions.get_mut(session_group).ok_or_else(|| { + TestkitError::Configuration(format!("adaptive segment has no prepared session {session_group}")) + })?; + if tokio::time::Instant::now() > session.expires_at { + return Err(TestkitError::Protocol("adaptive session TTL expired before segment activation".to_owned())); + } + session.state = AdaptiveSessionState::Active; + session.expires_at = tokio::time::Instant::now() + ttl; + Ok(session.state) + } + + pub fn preserve(&mut self, session_group: &str, ttl: Duration) -> Result<(), TestkitError> { + let session = self.sessions.get_mut(session_group).ok_or_else(|| { + TestkitError::Configuration(format!("cannot preserve unknown adaptive session {session_group}")) + })?; + session.state = AdaptiveSessionState::Preserved; + session.expires_at = tokio::time::Instant::now() + ttl; + Ok(()) + } + + pub fn reap_expired(&mut self) { + self.sessions.retain(|_, session| tokio::time::Instant::now() <= session.expires_at); + } + + #[must_use] + pub fn state(&self, session_group: &str) -> Option { + self.sessions.get(session_group).map(|session| session.state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn segment_activates_and_reacquisition_preserves_one_session() { + let mut tracker = AdaptiveSessionTracker::default(); + assert_eq!( + tracker.manifest("hls-a".to_owned(), Duration::from_secs(10)).unwrap(), + AdaptiveSessionState::Prepared + ); + assert_eq!(tracker.segment("hls-a", Duration::from_secs(10)).unwrap(), AdaptiveSessionState::Active); + tracker.preserve("hls-a", Duration::from_secs(10)).unwrap(); + assert_eq!(tracker.segment("hls-a", Duration::from_secs(10)).unwrap(), AdaptiveSessionState::Active); + } +} diff --git a/tools/tuliprox-testkit/src/bootstrap.rs b/tools/tuliprox-testkit/src/bootstrap.rs new file mode 100644 index 000000000..a3a162ae9 --- /dev/null +++ b/tools/tuliprox-testkit/src/bootstrap.rs @@ -0,0 +1,462 @@ +use crate::{ + config::{BootstrapConfig, Channel, PolicyContract}, + TestkitError, +}; +use std::{ + collections::HashMap, + fmt::Write, + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; +use tempfile::TempDir; +use tokio::process::{Child, Command}; + +const FIXTURE_PASSWORD: &str = "testkit-password"; + +/// Files owned by one isolated run. They are written below a unique temporary +/// directory and must never refer to an operator's configuration or storage. +pub struct FixturePaths { + _directory: TempDir, + pub root: PathBuf, + pub config_file: PathBuf, + pub source_file: PathBuf, + pub api_proxy_file: PathBuf, + pub web_root: PathBuf, +} + +impl FixturePaths { + pub fn create( + run_id: &str, + origin_address: SocketAddr, + tuliprox_address: SocketAddr, + policy: Option<&PolicyContract>, + channels: &HashMap, + ) -> Result { + let directory = tempfile::Builder::new().prefix("tuliprox-testkit-").tempdir()?; + let root = directory.path().to_path_buf(); + let web_root = root.join("web"); + let storage_dir = root.join("storage"); + let backup_dir = root.join("backup"); + let history_dir = root.join("history"); + std::fs::create_dir_all(web_root.join("static/.well-known"))?; + std::fs::create_dir_all(&storage_dir)?; + std::fs::create_dir_all(&backup_dir)?; + std::fs::create_dir_all(&history_dir)?; + + let config_file = root.join("config.yml"); + let source_file = root.join("source.yml"); + let api_proxy_file = root.join("api-proxy.yml"); + std::fs::write( + &config_file, + render_config(tuliprox_address, &web_root, &storage_dir, &backup_dir, &history_dir, policy)?, + )?; + std::fs::write(&source_file, render_sources(run_id, origin_address, channels, policy))?; + std::fs::write(&api_proxy_file, render_api_proxy(tuliprox_address, policy))?; + + Ok(Self { _directory: directory, root, config_file, source_file, api_proxy_file, web_root }) + } +} + +#[must_use] +pub const fn fixture_password() -> &'static str { FIXTURE_PASSWORD } + +fn render_config( + tuliprox_address: SocketAddr, + web_root: &Path, + storage_dir: &Path, + backup_dir: &Path, + history_dir: &Path, + policy: Option<&PolicyContract>, +) -> Result { + let user_access_control = policy.is_some_and(|contract| contract.user_access_control); + let grace = policy.and_then(|contract| contract.grace.as_ref()); + let grace_millis = grace.map_or(0, |contract| contract.timeout_millis); + let grace_hold = grace.is_none_or(|contract| matches!(contract.mode, crate::oracle::GraceMode::HoldStream)); + let mut config = format!( + "api:\n host: 127.0.0.1\n port: {}\n web_root: {}\nworking_dir: {}\nbackup_dir: {}\nuser_access_control: {user_access_control}\nupdate_on_boot: true\nlog:\n log_level: debug\nweb_ui:\n enabled: true\n user_ui_enabled: false\nreverse_proxy:\n stream:\n grace_period_millis: {grace_millis}\n grace_period_hold_stream: {grace_hold}\n shared_subscriber_idle_timeout_secs: 1\n", + tuliprox_address.port(), + yaml_scalar(web_root), + yaml_scalar(storage_dir), + yaml_scalar(backup_dir), + ); + if let Some(policy) = policy { + if let Some(strategies) = &policy.admission_strategies { + if strategies.is_empty() { + config.push_str(" admission_strategies: []\n"); + } else { + config.push_str(" admission_strategies:\n"); + for strategy in strategies { + let encoded = serde_json::to_string(strategy).map_err(|error| { + TestkitError::Configuration(format!("cannot render admission strategy: {error}")) + })?; + let _ = writeln!(config, " - {}", encoded.trim_matches('\"')); + } + } + } + } + let _ = writeln!( + config, + " stream_history:\n stream_history_enabled: true\n stream_history_directory: {}\n rewrite_secret: 0123456789abcdef0123456789abcdef", + yaml_scalar(history_dir) + ); + Ok(config) +} + +fn render_sources( + run_id: &str, + origin_address: SocketAddr, + channels: &HashMap, + policy: Option<&PolicyContract>, +) -> String { + let mut markers = channels.values().map(|channel| channel.origin_marker).collect::>(); + markers.sort_unstable(); + markers.dedup(); + if markers.is_empty() { + markers.push(17); + } + let marker_comment = markers.iter().map(u32::to_string).collect::>().join(","); + let provider_limit = policy + .and_then(|contract| contract.provider_max_connections) + .map_or_else(String::new, |limit| format!(" max_connections: {limit}\n")); + format!( + "inputs:\n - name: testkit-origin\n type: m3u\n url: http://{origin_address}/catalog/input.m3u?run={run_id}\n{provider_limit}sources:\n - inputs:\n - testkit-origin\n targets:\n - name: testkit\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n mpeg_ts: true\n# Origin markers for this fixture: {marker_comment}\n" + ) +} + +fn render_api_proxy(tuliprox_address: SocketAddr, policy: Option<&PolicyContract>) -> String { + let mut output = format!( + "server:\n - name: testkit\n protocol: http\n host: 127.0.0.1\n port: \"{}\"\n timezone: UTC\n message: Tuliprox testkit\nuser:\n - target: testkit\n credentials:\n", + tuliprox_address.port() + ); + if let Some(policy) = policy { + for (username, user) in &policy.users { + let _ = writeln!( + output, + " - username: {}\n password: {}\n proxy: reverse\n server: testkit\n max_connections: {}\n soft_connections: {}\n status: Active", + yaml_string(username), + FIXTURE_PASSWORD, + user.max_connections, + user.soft_connections, + ); + } + } + if !output.contains(" - username:") { + output.push_str(" - username: testkit\n password: testkit-password\n proxy: reverse\n server: testkit\n status: Active\n"); + } + output +} + +fn yaml_scalar(path: &Path) -> String { yaml_string(&path.to_string_lossy()) } + +fn yaml_string(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } + +/// Owns only the SUT process launched for this run and waits for its explicit readiness endpoint. +pub struct IsolatedSut { + child: Child, +} + +/// Owns every process and every generated file used by one self-contained +/// Tuliprox scenario. The fixture never adopts an externally running process. +pub struct IsolatedFixture { + pub paths: FixturePaths, + pub base_url: String, + pub origin_control_url: String, + origin: Child, + sut: IsolatedSut, +} + +impl IsolatedFixture { + #[allow(clippy::too_many_arguments)] + pub async fn start( + bootstrap: &BootstrapConfig, + working_directory: &Path, + testkit_binary: &Path, + run_id: &str, + origin_config: Option<&crate::config::OriginConfig>, + policy: Option<&PolicyContract>, + channels: &HashMap, + ) -> Result { + const MAX_START_ATTEMPTS: usize = 5; + let mut last_error = None; + + for attempt in 1..=MAX_START_ATTEMPTS { + let attempt_run_id = if attempt == 1 { run_id.to_string() } else { format!("{run_id}-retry{attempt}") }; + match Self::start_attempt( + bootstrap, + working_directory, + testkit_binary, + &attempt_run_id, + origin_config, + policy, + channels, + ) + .await + { + Ok(env) => { + if attempt > 1 { + eprintln!("Bootstrap succeeded on attempt {attempt}/{MAX_START_ATTEMPTS}"); + } + return Ok(env); + } + Err(error) => { + eprintln!("Bootstrap start attempt {attempt}/{MAX_START_ATTEMPTS} failed: {error}; retrying with fresh ports..."); + last_error = Some(error); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + + Err(last_error.unwrap_or_else(|| TestkitError::Protocol("failed to bootstrap environment".to_owned()))) + } + + #[allow(clippy::too_many_arguments)] + async fn start_attempt( + bootstrap: &BootstrapConfig, + working_directory: &Path, + testkit_binary: &Path, + run_id: &str, + origin_config: Option<&crate::config::OriginConfig>, + policy: Option<&PolicyContract>, + channels: &HashMap, + ) -> Result { + let origin_address = reserve_loopback_address()?; + let origin_control_address = reserve_loopback_address()?; + let tuliprox_address = reserve_loopback_address()?; + let paths = FixturePaths::create(run_id, origin_address, tuliprox_address, policy, channels)?; + let markers = fixture_markers(channels); + let mut origin_cmd = Command::new(testkit_binary); + origin_cmd + .arg("origin") + .arg("--listen") + .arg(origin_address.to_string()) + .arg("--control-listen") + .arg(origin_control_address.to_string()) + .arg("--run-id") + .arg(run_id) + .arg("--markers") + .arg(markers); + if let Some(origin_cfg) = origin_config { + if let Some(limit) = origin_cfg.account_limit { + origin_cmd.arg("--account-limit").arg(limit.to_string()); + } + if let Some(mode) = origin_cfg.limit_mode { + let mode_str = match mode { + crate::origin_transport::OriginLimitMode::ObserveOnly => "observe_only", + crate::origin_transport::OriginLimitMode::RejectNew => "reject_new", + crate::origin_transport::OriginLimitMode::EvictOldest => "evict_oldest", + }; + origin_cmd.arg("--limit-mode").arg(mode_str); + } + } + let mut origin = origin_cmd.current_dir(working_directory).spawn()?; + let origin_health = format!("http://{origin_control_address}/health"); + if let Err(error) = wait_for_http_ready(&mut origin, &origin_health, Duration::from_secs(10)).await { + let _ = stop_child(&mut origin).await; + return Err(error); + } + let base_url = format!("http://{tuliprox_address}"); + let rendered = render_bootstrap(bootstrap, &paths, &base_url)?; + let sut = match IsolatedSut::start(&rendered, working_directory).await { + Ok(sut) => sut, + Err(error) => { + let _ = stop_child(&mut origin).await; + return Err(error); + } + }; + let origin_control_url = format!("http://{origin_control_address}"); + Ok(Self { paths, base_url, origin_control_url, origin, sut }) + } + + pub async fn stop(mut self) -> Result<(), TestkitError> { + let sut_result = self.sut.stop().await; + let origin_result = stop_child(&mut self.origin).await; + match (sut_result, origin_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(sut_error), Err(origin_error)) => Err(TestkitError::Protocol(format!( + "Tuliprox cleanup failed ({sut_error}); origin cleanup failed ({origin_error})" + ))), + } + } +} + +fn reserve_loopback_address() -> Result { + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let address = listener.local_addr()?; + drop(listener); + Ok(address) +} + +fn fixture_markers(channels: &HashMap) -> String { + let mut markers = channels.values().map(|channel| channel.origin_marker).collect::>(); + markers.sort_unstable(); + markers.dedup(); + if markers.is_empty() { + markers.push(17); + } + markers.iter().map(u32::to_string).collect::>().join(",") +} + +fn render_bootstrap( + bootstrap: &BootstrapConfig, + paths: &FixturePaths, + base_url: &str, +) -> Result { + let render = |value: &str| { + value + .replace("{config_file}", &paths.config_file.to_string_lossy()) + .replace("{source_file}", &paths.source_file.to_string_lossy()) + .replace("{api_proxy_file}", &paths.api_proxy_file.to_string_lossy()) + .replace("{run_directory}", &paths.root.to_string_lossy()) + .replace("{base_url}", base_url) + }; + let command = resolve_environment_command(&render(&bootstrap.command))?; + Ok(BootstrapConfig { + command, + arguments: bootstrap.arguments.iter().map(|argument| render(argument)).collect(), + readiness_url: render(&bootstrap.readiness_url), + readiness_timeout_millis: bootstrap.readiness_timeout_millis, + }) +} + +fn resolve_environment_command(command: &str) -> Result { + let Some(variable) = command.strip_prefix("${env:").and_then(|value| value.strip_suffix('}')) else { + return Ok(command.to_owned()); + }; + std::env::var(variable).map_err(|_| { + TestkitError::Configuration(format!("bootstrap command environment variable {variable} is not set")) + }) +} + +async fn wait_for_http_ready(child: &mut Child, readiness_url: &str, timeout: Duration) -> Result<(), TestkitError> { + let deadline = tokio::time::Instant::now() + timeout; + let client = reqwest::Client::new(); + loop { + if let Some(status) = child.try_wait()? { + return Err(TestkitError::Protocol(format!("fixture process exited before readiness: {status}"))); + } + if client.get(readiness_url).send().await.is_ok_and(|response| response.status().is_success()) { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!("fixture process did not become ready at {readiness_url}"))); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn stop_child(child: &mut Child) -> Result<(), TestkitError> { + if child.try_wait()?.is_some() { + return Ok(()); + } + child.start_kill()?; + tokio::time::timeout(Duration::from_secs(5), child.wait()) + .await + .map_err(|_| TestkitError::Protocol("fixture process did not exit during cleanup".to_owned()))??; + Ok(()) +} + +impl IsolatedSut { + pub async fn start(config: &BootstrapConfig, working_directory: &Path) -> Result { + if config.command.trim().is_empty() || config.readiness_url.trim().is_empty() { + return Err(TestkitError::Configuration("bootstrap command and readiness URL are required".to_owned())); + } + let child = Command::new(&config.command).args(&config.arguments).current_dir(working_directory).spawn()?; + let mut sut = Self { child }; + if let Err(startup_error) = + sut.wait_ready(&config.readiness_url, Duration::from_millis(config.readiness_timeout_millis.max(1))).await + { + if let Err(cleanup_error) = sut.stop().await { + return Err(TestkitError::Protocol(format!( + "isolated Tuliprox startup failed ({startup_error}) and cleanup failed ({cleanup_error})" + ))); + } + return Err(startup_error); + } + Ok(sut) + } + + async fn wait_ready(&mut self, readiness_url: &str, timeout: Duration) -> Result<(), TestkitError> { + let deadline = tokio::time::Instant::now() + timeout; + let client = reqwest::Client::new(); + loop { + if let Some(status) = self.child.try_wait()? { + return Err(TestkitError::Protocol(format!( + "isolated Tuliprox process exited before readiness: {status}" + ))); + } + if let Ok(response) = client.get(readiness_url).send().await { + if response.status().is_success() { + return Ok(()); + } + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol("isolated Tuliprox process did not become ready".to_owned())); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + pub async fn stop(mut self) -> Result<(), TestkitError> { + if self.child.try_wait()?.is_none() { + self.child.start_kill()?; + self.child.wait().await?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn early_process_exit_does_not_wait_for_the_readiness_deadline() { + let config = BootstrapConfig { + command: "/bin/false".to_owned(), + arguments: Vec::new(), + readiness_url: "http://127.0.0.1:1/ready".to_owned(), + readiness_timeout_millis: 5_000, + }; + let started_at = tokio::time::Instant::now(); + let result = IsolatedSut::start(&config, Path::new("/tmp")).await; + assert!(matches!(result, Err(TestkitError::Protocol(message)) if message.contains("exited before readiness"))); + assert!(started_at.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn fixture_files_are_run_scoped_and_contain_no_operator_paths() -> Result<(), TestkitError> { + let policy = PolicyContract { + user_access_control: true, + users: HashMap::from([( + "fixture-user".to_owned(), + crate::config::UserPolicy { max_connections: 1, soft_connections: 0 }, + )]), + admission_strategies: Some(vec![crate::oracle::AdmissionStrategy::EvictUserSameIpLatest]), + grace: None, + provider_max_connections: None, + expected_provider_slots: None, + }; + let channels = HashMap::from([( + "fixture-channel".to_owned(), + Channel { origin_marker: 17, protocol: "xtream_ts".to_owned() }, + )]); + let fixture = FixturePaths::create( + "fixture-run", + "127.0.0.1:9910".parse::().map_err(|error| TestkitError::Configuration(error.to_string()))?, + "127.0.0.1:8901".parse::().map_err(|error| TestkitError::Configuration(error.to_string()))?, + Some(&policy), + &channels, + )?; + let config = std::fs::read_to_string(&fixture.config_file)?; + let source = std::fs::read_to_string(&fixture.source_file)?; + let api_proxy = std::fs::read_to_string(&fixture.api_proxy_file)?; + + assert!(config.contains(fixture.root.to_string_lossy().as_ref())); + assert!(source.contains("fixture-run")); + assert!(api_proxy.contains("fixture-user")); + assert!(!config.contains("/home/")); + Ok(()) + } +} diff --git a/tools/tuliprox-testkit/src/config.rs b/tools/tuliprox-testkit/src/config.rs new file mode 100644 index 000000000..da508dd4a --- /dev/null +++ b/tools/tuliprox-testkit/src/config.rs @@ -0,0 +1,607 @@ +use crate::{ + oracle::{AdmissionStrategy, GraceMode}, + TestkitError, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, HashSet}, + fs, + path::Path, + time::Duration, +}; + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scenario { + pub schema_version: u16, + pub name: String, + pub tuliprox: TuliproxConfig, + #[serde(default)] + pub origin: Option, + #[serde(default)] + pub actors: Vec, + #[serde(default)] + pub channels: HashMap, + #[serde(default)] + pub agents: AgentsConfig, + #[serde(default)] + pub policy_contract: Option, + #[serde(default)] + pub steps: Vec, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AgentsConfig { + #[serde(default)] + pub required: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyContract { + pub user_access_control: bool, + #[serde(default)] + pub users: HashMap, + /// `None` preserves an omitted field, which has distinct Tuliprox semantics + /// from an explicit empty admission-strategy list. + #[serde(default)] + pub admission_strategies: Option>, + #[serde(default)] + pub grace: Option, + #[serde(default)] + pub provider_max_connections: Option, + #[serde(default)] + pub expected_provider_slots: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UserPolicy { + pub max_connections: usize, + #[serde(default)] + pub soft_connections: usize, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GraceContract { + pub mode: GraceMode, + pub timeout_millis: u64, +} + +impl PolicyContract { + #[must_use] + pub fn effective_admission_strategies(&self) -> Vec { + self.admission_strategies.clone().unwrap_or_else(|| { + self.grace.as_ref().map_or_else(Vec::new, |grace| match grace.mode { + GraceMode::Instant => vec![AdmissionStrategy::GraceInstantStream], + GraceMode::HoldStream => vec![AdmissionStrategy::GraceHoldStream], + }) + }) + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TuliproxConfig { + pub base_url: String, + #[serde(default)] + pub api_base_url: Option, + #[serde(default)] + pub admin_username: Option, + #[serde(default)] + pub admin_password_env: Option, + #[serde(default)] + pub playlist_url: Option, + #[serde(default)] + pub bootstrap: Option, + #[serde(default)] + pub execution_mode: ExecutionMode, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OriginConfig { + pub run_id: String, + #[serde(default)] + pub account_limit: Option, + #[serde(default)] + pub limit_mode: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BootstrapConfig { + pub command: String, + #[serde(default)] + pub arguments: Vec, + pub readiness_url: String, + #[serde(default = "default_readiness_timeout_millis")] + pub readiness_timeout_millis: u64, +} + +fn default_readiness_timeout_millis() -> u64 { 30_000 } + +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionMode { + #[default] + IsolatedFixture, + ExistingInstance, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Actor { + pub id: String, + pub agent: String, + #[serde(default)] + pub username: Option, + #[serde(default)] + pub password_env: Option, + #[serde(default)] + pub user_agent: Option, + #[serde(default)] + pub client_ip: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClientIp { + pub mode: String, + #[serde(default)] + pub value: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Channel { + pub origin_marker: u32, + pub protocol: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Step { + pub command_id: String, + #[serde(default)] + pub expect: ExpectedPlayback, + pub start: Option, + #[serde(default)] + pub stop: Option, + #[serde(default)] + pub repeat: Option, + #[serde(default)] + pub await_frames: Option, + #[serde(default, rename = "await")] + pub await_condition: Option, + #[serde(default)] + pub assert_origin: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ExpectedRejection { + #[serde(default)] + pub custom_video: Option, + #[serde(default)] + pub http_status: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExpectedPlayback { + #[default] + Streaming, + Rejected, + RejectedWith(ExpectedRejection), +} + +impl ExpectedPlayback { + #[must_use] + pub fn is_streaming(&self) -> bool { matches!(self, Self::Streaming) } + + #[must_use] + pub fn is_rejected(&self) -> bool { matches!(self, Self::Rejected | Self::RejectedWith(_)) } + + #[must_use] + pub fn matches_outcome(&self, outcome: &crate::protocol::PlaybackOutcome) -> bool { + use crate::protocol::{PlaybackOutcome, RejectionReason}; + match (self, outcome) { + (Self::Streaming, PlaybackOutcome::Streaming { .. }) + | (Self::Rejected, PlaybackOutcome::AdmissionRejected { .. }) => true, + (Self::RejectedWith(expected), PlaybackOutcome::AdmissionRejected { reason }) => match reason { + RejectionReason::CustomVideo(kind) => { + expected.custom_video.is_none_or(|expected_kind| expected_kind == *kind) + } + RejectionReason::HttpStatus(status) => { + expected.http_status.is_none_or(|expected_status| expected_status == *status) + } + _ => expected.custom_video.is_none() && expected.http_status.is_none(), + }, + _ => false, + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AwaitCondition { + #[serde(default)] + pub valid_frames: Option, +} + +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct AssertOrigin { + #[serde(default)] + pub max_active_tcp: Option, + #[serde(default)] + pub max_active_body: Option, + #[serde(default)] + pub active_body: Option, + #[serde(default)] + pub no_evictions: Option, + #[serde(default)] + pub no_limit_rejections: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum PostReadAction { + #[default] + KeepOpen, + Pause, + Close, +} + +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct Start { + pub actor: String, + pub playback_id: String, + pub session_group: String, + #[serde(default)] + pub url: Option, + #[serde(default)] + pub channel: Option, + #[serde(default)] + pub method: Option, + #[serde(default)] + pub range: Option, + #[serde(default)] + pub expected_status: Option, + #[serde(default)] + pub expected_content_range: Option, + #[serde(default)] + pub read_limit_bytes: Option, + #[serde(default)] + pub post_read_action: Option, + #[serde(default)] + pub user_agent: Option, + #[serde(default)] + pub vod_object: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Stop { + pub playback_id: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Repeat { + pub count: usize, + pub steps: Vec, +} + +impl Scenario { + pub fn from_path(path: &Path) -> Result { + let content = fs::read_to_string(path)?; + let scenario: Self = + serde_saphyr::from_str(&content).map_err(|error| TestkitError::Configuration(error.to_string()))?; + scenario.validate()?; + Ok(scenario) + } + + pub fn validate(&self) -> Result<(), TestkitError> { + if self.schema_version != 1 { + return Err(TestkitError::Configuration("unsupported scenario schema version".to_owned())); + } + if self.name.trim().is_empty() || self.tuliprox.base_url.trim().is_empty() { + return Err(TestkitError::Configuration("scenario name and Tuliprox URL are required".to_owned())); + } + if self.tuliprox.execution_mode == ExecutionMode::IsolatedFixture && self.tuliprox.bootstrap.is_none() { + return Err(TestkitError::Configuration("isolated_fixture requires a bootstrap definition".to_owned())); + } + if self.origin.as_ref().is_some_and(|origin| origin.run_id.trim().is_empty()) { + return Err(TestkitError::Configuration("origin run_id cannot be empty".to_owned())); + } + if self.policy_contract.as_ref().and_then(|contract| contract.provider_max_connections) == Some(0) { + return Err(TestkitError::Configuration( + "provider_max_connections must be positive when specified".to_owned(), + )); + } + let actor_ids = self.actors.iter().map(|actor| actor.id.as_str()).collect::>(); + if actor_ids.len() != self.actors.len() || actor_ids.iter().any(|id| id.is_empty()) { + return Err(TestkitError::Configuration("actor IDs must be unique and non-empty".to_owned())); + } + let steps = self.expanded_steps()?; + let mut command_ids = HashSet::new(); + let mut playback_ids = HashSet::new(); + let mut seen_playback_ids = HashSet::new(); + for step in &steps { + if !command_ids.insert(step.command_id.as_str()) || step.command_id.is_empty() { + return Err(TestkitError::Configuration("command IDs must be unique and non-empty".to_owned())); + } + if step.start.is_some() == step.stop.is_some() { + return Err(TestkitError::Configuration( + "each step must contain exactly one of start or stop".to_owned(), + )); + } + let Some(start) = &step.start else { + let stop = step.stop.as_ref().ok_or_else(|| TestkitError::Configuration("missing stop".to_owned()))?; + if stop.playback_id.is_empty() || !playback_ids.contains(stop.playback_id.as_str()) { + return Err(TestkitError::Configuration(format!( + "stop references inactive playback {}", + stop.playback_id + ))); + } + playback_ids.remove(stop.playback_id.as_str()); + continue; + }; + if !actor_ids.contains(start.actor.as_str()) { + return Err(TestkitError::Configuration(format!("unknown actor {}", start.actor))); + } + if start.playback_id.is_empty() || start.session_group.is_empty() { + return Err(TestkitError::Configuration("start requires playback_id and session_group".to_owned())); + } + let target_count = [start.url.is_some(), start.channel.is_some(), start.vod_object.is_some()] + .into_iter() + .filter(|&v| v) + .count(); + if target_count != 1 { + return Err(TestkitError::Configuration( + "start requires exactly one of url, channel, or vod_object".to_owned(), + )); + } + if start.url.as_deref().is_some_and(str::is_empty) + || start.channel.as_deref().is_some_and(str::is_empty) + || start.vod_object.as_deref().is_some_and(str::is_empty) + { + return Err(TestkitError::Configuration( + "start URL, channel, or vod_object cannot be empty".to_owned(), + )); + } + if let Some(channel) = &start.channel { + if !self.channels.contains_key(channel) { + return Err(TestkitError::Configuration(format!("unknown channel {channel}"))); + } + } + if !seen_playback_ids.insert(start.playback_id.as_str()) { + return Err(TestkitError::Configuration(format!("duplicate playback ID {}", start.playback_id))); + } + playback_ids.insert(start.playback_id.as_str()); + if step.expect == ExpectedPlayback::Rejected + && step + .await_frames + .or_else(|| step.await_condition.as_ref().and_then(|condition| condition.valid_frames)) + .is_some() + { + return Err(TestkitError::Configuration("rejected playback must not await valid frames".to_owned())); + } + } + if let Some(contract) = &self.policy_contract { + if !contract.user_access_control && !contract.users.is_empty() { + return Err(TestkitError::Configuration("policy users require user_access_control".to_owned())); + } + if contract.grace.as_ref().is_some_and(|grace| grace.timeout_millis == 0) { + return Err(TestkitError::Configuration("grace timeout must be non-zero".to_owned())); + } + } + Ok(()) + } + + pub fn expanded_steps(&self) -> Result, TestkitError> { + let mut expanded = Vec::new(); + expand_steps(&self.steps, "", &mut expanded)?; + Ok(expanded) + } +} + +fn expand_steps(steps: &[Step], prefix: &str, expanded: &mut Vec) -> Result<(), TestkitError> { + const MAX_REPEAT_COUNT: usize = 1_000; + for step in steps { + if step.command_id.is_empty() { + return Err(TestkitError::Configuration("command IDs must be unique and non-empty".to_owned())); + } + if let Some(repeat) = &step.repeat { + if step.start.is_some() || step.stop.is_some() { + return Err(TestkitError::Configuration("repeat cannot be combined with start or stop".to_owned())); + } + if repeat.count == 0 || repeat.count > MAX_REPEAT_COUNT || repeat.steps.is_empty() { + return Err(TestkitError::Configuration("repeat requires 1 through 1000 child steps".to_owned())); + } + for iteration in 0..repeat.count { + expand_steps(&repeat.steps, &format!("{prefix}{}:{iteration}:", step.command_id), expanded)?; + } + continue; + } + if step.start.is_some() == step.stop.is_some() { + return Err(TestkitError::Configuration("each step must contain exactly one of start or stop".to_owned())); + } + let mut child = step.clone(); + child.command_id = format!("{prefix}{}", step.command_id); + if let Some(start) = child.start.as_mut() { + start.playback_id = format!("{prefix}{}", start.playback_id); + start.session_group = format!("{prefix}{}", start.session_group); + } + if let Some(stop) = child.stop.as_mut() { + stop.playback_id = format!("{prefix}{}", stop.playback_id); + } + expanded.push(child); + } + Ok(()) +} + +#[must_use] +pub fn lease_duration(millis: u64) -> Duration { Duration::from_millis(millis.max(1)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn duplicate_playbacks_are_rejected() { + let scenario = Scenario { + schema_version: 1, + name: "test".to_owned(), + tuliprox: TuliproxConfig { + base_url: "http://example.invalid".to_owned(), + api_base_url: None, + admin_username: None, + admin_password_env: None, + playlist_url: None, + bootstrap: None, + execution_mode: ExecutionMode::IsolatedFixture, + }, + origin: None, + actors: vec![Actor { + id: "a".to_owned(), + agent: "local".to_owned(), + username: None, + password_env: None, + user_agent: None, + client_ip: None, + }], + channels: HashMap::new(), + agents: AgentsConfig::default(), + policy_contract: None, + steps: (0..2) + .map(|number| Step { + command_id: number.to_string(), + expect: ExpectedPlayback::Streaming, + start: Some(Start { + actor: "a".to_owned(), + playback_id: "same".to_owned(), + session_group: "s".to_owned(), + url: Some("http://example.invalid".to_owned()), + channel: None, + ..Default::default() + }), + stop: None, + repeat: None, + await_frames: None, + await_condition: None, + assert_origin: None, + }) + .collect(), + }; + assert!(scenario.validate().is_err()); + } + + #[test] + fn checked_in_admission_scenarios_match_the_versioned_schema() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test/fixtures/testkit/scenarios"); + for filename in [ + "hard-limit.yml", + "evict-same-ip-latest.yml", + "evict-same-ip-oldest.yml", + "evict-user-oldest.yml", + "evict-user-latest.yml", + "provider-hard-limit.yml", + "shared-stream-single-provider-slot.yml", + "vod-range-reopen-preserves-three-live.yml", + "vod-range-reopen-strict-cap.yml", + "vod-reopen-backpressured-body.yml", + ] { + assert!(Scenario::from_path(&root.join(filename)).is_ok(), "{filename}"); + } + } + + #[test] + fn stop_must_reference_an_active_playback() { + let scenario: Scenario = serde_saphyr::from_str( + r" +schema_version: 1 +name: stop-validation +tuliprox: { base_url: http://example.invalid, execution_mode: existing_instance } +actors: [{ id: a, agent: local }] +steps: + - command_id: stop-missing + stop: { playback_id: missing } +", + ) + .unwrap(); + assert!(scenario.validate().is_err()); + } + + #[test] + fn repeat_derives_distinct_playback_and_command_ids_per_iteration() { + let scenario: Scenario = serde_saphyr::from_str( + r" +schema_version: 1 +name: repeat-validation +tuliprox: { base_url: http://example.invalid, execution_mode: existing_instance } +actors: [{ id: a, agent: local }] +steps: + - command_id: repeat + repeat: + count: 2 + steps: + - command_id: begin + start: { actor: a, playback_id: playback, session_group: session, url: http://example.invalid/17.ts } + - command_id: end + stop: { playback_id: playback } +", + ) + .unwrap(); + assert!(scenario.validate().is_ok()); + let expanded = scenario.expanded_steps().unwrap(); + assert_eq!(expanded.len(), 4); + assert_eq!(expanded[0].command_id, "repeat:0:begin"); + assert_eq!(expanded[0].start.as_ref().map(|start| start.playback_id.as_str()), Some("repeat:0:playback")); + assert_eq!(expanded[2].command_id, "repeat:1:begin"); + assert_eq!(expanded[3].stop.as_ref().map(|stop| stop.playback_id.as_str()), Some("repeat:1:playback")); + } + + #[test] + fn rejected_playback_does_not_match_500_corrupt_data_or_early_eof() { + use crate::{ + custom_video::CustomVideoKind, + protocol::{PlaybackOutcome, RejectionReason}, + }; + + let expected_rejected = ExpectedPlayback::Rejected; + + // Proved admission rejection matches + let valid_cv = PlaybackOutcome::AdmissionRejected { + reason: RejectionReason::CustomVideo(CustomVideoKind::UserConnectionsExhausted), + }; + assert!(expected_rejected.matches_outcome(&valid_cv)); + + let valid_http = PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(429) }; + assert!(expected_rejected.matches_outcome(&valid_http)); + + // HTTP 500 MUST NOT pass as expected rejection + let http_500 = PlaybackOutcome::HttpError { status: 500 }; + assert!(!expected_rejected.matches_outcome(&http_500)); + + // Invalid marker/corrupt frame MUST NOT pass + let invalid_marker = PlaybackOutcome::InvalidData { message: "wrong marker".to_owned() }; + assert!(!expected_rejected.matches_outcome(&invalid_marker)); + + // Early EOF MUST NOT pass + let early_eof = PlaybackOutcome::UnexpectedEof { frames: 0, bytes: 0 }; + assert!(!expected_rejected.matches_outcome(&early_eof)); + + // Transport error MUST NOT pass + let transport_err = PlaybackOutcome::TransportError { message: "connection reset".to_owned() }; + assert!(!expected_rejected.matches_outcome(&transport_err)); + + // Idle timeout MUST NOT pass + assert!(!expected_rejected.matches_outcome(&PlaybackOutcome::IdleTimeout)); + } +} diff --git a/tools/tuliprox-testkit/src/control.rs b/tools/tuliprox-testkit/src/control.rs new file mode 100644 index 000000000..ec0368166 --- /dev/null +++ b/tools/tuliprox-testkit/src/control.rs @@ -0,0 +1,63 @@ +use crate::{ + protocol::{PlaybackId, RunId}, + TestkitError, +}; +use std::collections::HashMap; + +/// Tracks the semantic start identity for a run independently from transport message IDs. +/// A replay with identical parameters is harmless; a conflicting replay is rejected. +#[derive(Debug, Default)] +pub struct PlaybackRegistry { + starts: HashMap<(RunId, u64, PlaybackId), String>, +} + +impl PlaybackRegistry { + pub fn register_start( + &mut self, + run_id: RunId, + generation: u64, + playback_id: PlaybackId, + fingerprint: String, + ) -> Result { + let key = (run_id, generation, playback_id); + match self.starts.get(&key) { + None => { + self.starts.insert(key, fingerprint); + Ok(StartDisposition::New) + } + Some(existing) if existing == &fingerprint => Ok(StartDisposition::Replay), + Some(_) => Err(TestkitError::Protocol("playback ID was reused with different start parameters".to_owned())), + } + } + + pub fn stop(&mut self, run_id: &RunId, generation: u64, playback_id: &PlaybackId) -> bool { + self.starts.remove(&(run_id.clone(), generation, playback_id.clone())).is_some() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StartDisposition { + New, + Replay, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replay_is_safe_but_conflict_is_rejected() { + let mut registry = PlaybackRegistry::default(); + let run = RunId::new("run"); + let playback = PlaybackId::new("playback"); + assert!(matches!( + registry.register_start(run.clone(), 1, playback.clone(), "first".to_owned()), + Ok(StartDisposition::New) + )); + assert!(matches!( + registry.register_start(run.clone(), 1, playback.clone(), "first".to_owned()), + Ok(StartDisposition::Replay) + )); + assert!(registry.register_start(run, 1, playback, "different".to_owned()).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/custom_video.rs b/tools/tuliprox-testkit/src/custom_video.rs new file mode 100644 index 000000000..fec373bd0 --- /dev/null +++ b/tools/tuliprox-testkit/src/custom_video.rs @@ -0,0 +1,124 @@ +use crate::TestkitError; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::Path}; + +const PREFIX: &[u8] = b"TPX-CVS-"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CustomVideoKind { + ChannelUnavailable, + UserConnectionsExhausted, + ProviderConnectionsExhausted, + LowPriorityPreempted, + UserAccountExpired, + PanelApiProvisioning, + HlsSessionOrLeaseExpired, +} + +impl CustomVideoKind { + fn parse(value: &str) -> Option { + match value { + "CHANNEL_UNAVAILABLE" => Some(Self::ChannelUnavailable), + "USER_CONNECTIONS_EXHAUSTED" => Some(Self::UserConnectionsExhausted), + "PROVIDER_CONNECTIONS_EXHAUSTED" => Some(Self::ProviderConnectionsExhausted), + "LOW_PRIORITY_PREEMPTED" => Some(Self::LowPriorityPreempted), + "USER_ACCOUNT_EXPIRED" => Some(Self::UserAccountExpired), + "PANEL_API_PROVISIONING" => Some(Self::PanelApiProvisioning), + "HLS_SESSION_OR_LEASE_EXPIRED" => Some(Self::HlsSessionOrLeaseExpired), + _ => None, + } + } +} + +#[derive(Debug, Default)] +pub struct WireMarkerScanner { + retained: Vec, +} + +impl WireMarkerScanner { + pub fn push(&mut self, bytes: &[u8]) -> Result, TestkitError> { + self.retained.extend_from_slice(bytes); + if let Some(position) = self.retained.windows(PREFIX.len()).position(|window| window == PREFIX) { + let value_start = position + PREFIX.len(); + let Some(value_end) = + self.retained[value_start..].iter().position(|byte| !matches!(byte, b'A'..=b'Z' | b'_')) + else { + self.limit_buffer(); + return Ok(None); + }; + let marker = std::str::from_utf8(&self.retained[value_start..value_start + value_end]) + .map_err(|error| TestkitError::Protocol(format!("invalid custom-video marker: {error}")))?; + let kind = CustomVideoKind::parse(marker) + .ok_or_else(|| TestkitError::Protocol(format!("unknown custom-video marker {marker}")))?; + self.retained.clear(); + return Ok(Some(kind)); + } + self.limit_buffer(); + Ok(None) + } + + fn limit_buffer(&mut self) { + const MAX_RETAINED: usize = 128; + if self.retained.len() > MAX_RETAINED { + let start = self.retained.len() - MAX_RETAINED; + self.retained.drain(..start); + } + } +} + +#[must_use] +pub fn fixture_digest(bytes: &[u8]) -> String { blake3::hash(bytes).to_hex().to_string() } + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FixtureManifest { + pub schema_version: u16, + pub fixtures: BTreeMap, +} + +/// Verifies fixture installation bytes only. It must never be used to classify +/// bytes observed after Tuliprox has processed a transport stream. +pub fn verify_fixture_manifest(root: &Path, manifest: &FixtureManifest) -> Result<(), TestkitError> { + if manifest.schema_version != 1 { + return Err(TestkitError::Configuration("unsupported custom-video fixture manifest schema".to_owned())); + } + for (relative_path, expected_digest) in &manifest.fixtures { + let path = Path::new(relative_path); + if path.is_absolute() || path.components().any(|component| matches!(component, std::path::Component::ParentDir)) + { + return Err(TestkitError::Configuration(format!("unsafe fixture path {relative_path}"))); + } + let bytes = std::fs::read(root.join(path))?; + let actual_digest = fixture_digest(&bytes); + if actual_digest != *expected_digest { + return Err(TestkitError::Protocol(format!("fixture digest mismatch for {relative_path}"))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recognizes_marker_split_across_transport_chunks() { + let mut scanner = WireMarkerScanner::default(); + assert!(scanner.push(b"ignored TPX-CVS-USER_").unwrap().is_none()); + assert_eq!(scanner.push(b"CONNECTIONS_EXHAUSTED;").unwrap(), Some(CustomVideoKind::UserConnectionsExhausted)); + } + + #[test] + fn fixture_digest_is_deterministic() { + assert_eq!(fixture_digest(b"fixture"), fixture_digest(b"fixture")); + } + + #[test] + fn checked_in_fixture_manifest_matches_installation_bytes() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test/fixtures"); + let manifest_path = root.join("testkit/custom-video-fixtures.json"); + let manifest: FixtureManifest = serde_json::from_slice(&std::fs::read(manifest_path).unwrap()).unwrap(); + assert!(verify_fixture_manifest(&root, &manifest).is_ok()); + } +} diff --git a/tools/tuliprox-testkit/src/discovery.rs b/tools/tuliprox-testkit/src/discovery.rs new file mode 100644 index 000000000..bb4c519cc --- /dev/null +++ b/tools/tuliprox-testkit/src/discovery.rs @@ -0,0 +1,78 @@ +use crate::TestkitError; +use std::collections::HashMap; + +/// Maps a stable synthetic-origin marker to the virtual playback URL published by Tuliprox. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VirtualIdMap { + urls: HashMap, + named_urls: HashMap, +} + +impl VirtualIdMap { + pub fn from_m3u(document: &str) -> Result { + let mut urls = HashMap::new(); + let mut named_urls = HashMap::new(); + let mut pending_marker = None; + let mut pending_name = None; + for raw_line in document.lines() { + let line = raw_line.trim(); + if line.starts_with("#EXTINF:") { + pending_marker = marker_from_extinf(line); + pending_name = name_from_extinf(line); + continue; + } + if line.is_empty() || line.starts_with('#') { + continue; + } + if let Some(marker) = pending_marker.take() { + if urls.insert(marker, line.to_owned()).is_some() { + return Err(TestkitError::Configuration(format!("duplicate origin marker {marker} in playlist"))); + } + } + if let Some(name) = pending_name.take() { + named_urls.insert(name, line.to_owned()); + } + } + if urls.is_empty() && named_urls.is_empty() { + return Err(TestkitError::Configuration("playlist has no testkit marker entries".to_owned())); + } + Ok(Self { urls, named_urls }) + } + + pub fn playback_url(&self, marker: u32) -> Result<&str, TestkitError> { + self.urls + .get(&marker) + .map(String::as_str) + .ok_or_else(|| TestkitError::Configuration(format!("origin marker {marker} was not published by Tuliprox"))) + } + + pub fn named_playback_url(&self, name: &str) -> Result<&str, TestkitError> { + self.named_urls + .get(name) + .map(String::as_str) + .ok_or_else(|| TestkitError::Configuration(format!("named media {name} was not published by Tuliprox"))) + } +} + +fn marker_from_extinf(line: &str) -> Option { + let value = line.split("tvg-id=\"").nth(1)?.split('"').next()?; + value.strip_prefix("test-")?.parse().ok() +} + +fn name_from_extinf(line: &str) -> Option { + let value = line.split("tvg-id=\"").nth(1)?.split('"').next()?; + value.strip_prefix("test-vod-").map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovers_virtual_playback_urls_without_assuming_their_ids() { + let playlist = "#EXTM3U\n#EXTINF:-1 tvg-id=\"test-17\",News\nhttp://tuliprox/m3u/abc/17\n#EXTINF:-1 tvg-id=\"test-vod-movie.mkv\" type=\"movie\",Movie\nhttp://tuliprox/movie/abc/def/1.mkv\n"; + let map = VirtualIdMap::from_m3u(playlist).unwrap(); + assert_eq!(map.playback_url(17).unwrap(), "http://tuliprox/m3u/abc/17"); + assert_eq!(map.named_playback_url("movie.mkv").unwrap(), "http://tuliprox/movie/abc/def/1.mkv"); + } +} diff --git a/tools/tuliprox-testkit/src/faults.rs b/tools/tuliprox-testkit/src/faults.rs new file mode 100644 index 000000000..30275a896 --- /dev/null +++ b/tools/tuliprox-testkit/src/faults.rs @@ -0,0 +1,50 @@ +use crate::TestkitError; +use std::collections::HashMap; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OriginFault { + CloseBeforeFirstByte, + CloseAfterBytes(u64), + StallAfterBytes(u64), + CorruptNextFrame, + UnexpectedMarker(u32), +} + +#[derive(Debug, Default)] +pub struct FaultSchedule { + faults: HashMap, +} + +impl FaultSchedule { + pub fn insert(&mut self, fault_id: String, fault: OriginFault) -> Result<(), TestkitError> { + if self.faults.contains_key(&fault_id) { + return Err(TestkitError::Configuration(format!("fault ID {fault_id} already exists"))); + } + self.faults.insert(fault_id, fault); + Ok(()) + } + + pub fn take(&mut self, fault_id: &str) -> Option { self.faults.remove(fault_id) } + + pub fn clear(&mut self, fault_id: &str) -> Result<(), TestkitError> { + self.faults + .remove(fault_id) + .map(|_| ()) + .ok_or_else(|| TestkitError::Configuration(format!("fault ID {fault_id} does not exist"))) + } + + pub fn clear_all(&mut self) { self.faults.clear(); } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fault_is_consumed_only_once() { + let mut schedule = FaultSchedule::default(); + schedule.insert("close".to_owned(), OriginFault::CloseBeforeFirstByte).unwrap(); + assert_eq!(schedule.take("close"), Some(OriginFault::CloseBeforeFirstByte)); + assert_eq!(schedule.take("close"), None); + } +} diff --git a/tools/tuliprox-testkit/src/frame.rs b/tools/tuliprox-testkit/src/frame.rs new file mode 100644 index 000000000..49c43eb54 --- /dev/null +++ b/tools/tuliprox-testkit/src/frame.rs @@ -0,0 +1,251 @@ +use crate::{ + protocol::{OriginStreamId, RunId}, + TestkitError, +}; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use crc32fast::hash; + +pub const MAGIC: [u8; 8] = *b"TPXTEST\0"; +const VERSION: u8 = 1; +const FIXED_HEADER_LEN: usize = 8 + 1 + 1 + 2 + 16 + 16 + 4 + 8 + 8 + 4 + 4 + 4; +const FIXED_HEADER_LEN_U16: u16 = 76; +const TRAILER_LEN: usize = 4; +pub const MAX_PAYLOAD_LEN: usize = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + pub run_id: [u8; 16], + pub origin_stream_id: [u8; 16], + pub channel_marker: u32, + pub sequence: u64, + pub origin_mono_nanos: u64, + pub configured_bps: u32, + pub payload: Bytes, +} + +impl Frame { + #[must_use] + pub fn synthetic(run_id: &RunId, stream_id: &OriginStreamId, marker: u32, sequence: u64) -> Self { + let mut run = [0_u8; 16]; + let mut stream = [0_u8; 16]; + run.copy_from_slice(&blake3::hash(run_id.0.as_bytes()).as_bytes()[..16]); + stream.copy_from_slice(&blake3::hash(stream_id.0.as_bytes()).as_bytes()[..16]); + let payload = format!("marker={marker};sequence={sequence}\n").into_bytes(); + Self { + run_id: run, + origin_stream_id: stream, + channel_marker: marker, + sequence, + origin_mono_nanos: 0, + configured_bps: 64_000, + payload: Bytes::from(payload), + } + } + + pub fn encode(&self, output: &mut BytesMut) -> Result<(), TestkitError> { + let payload_len = self.payload.len(); + if payload_len > MAX_PAYLOAD_LEN { + return Err(TestkitError::Protocol("frame payload exceeds configured limit".to_owned())); + } + let start = output.len(); + output.reserve(FIXED_HEADER_LEN + payload_len + TRAILER_LEN); + output.put_slice(&MAGIC); + output.put_u8(VERSION); + output.put_u8(0); + output.put_u16(FIXED_HEADER_LEN_U16); + output.put_slice(&self.run_id); + output.put_slice(&self.origin_stream_id); + output.put_u32(self.channel_marker); + output.put_u64(self.sequence); + output.put_u64(self.origin_mono_nanos); + output.put_u32(self.configured_bps); + output.put_u32( + u32::try_from(payload_len) + .map_err(|_| TestkitError::Protocol("frame payload length cannot be encoded".to_owned()))?, + ); + let header_crc = hash(&output[start..]); + output.put_u32(header_crc); + output.put_slice(&self.payload); + output.put_u32(hash(&self.payload)); + Ok(()) + } +} + +#[derive(Debug, Default)] +pub struct FrameDecoder { + buffer: BytesMut, +} + +/// Verifies that decoded frames belong to one requested synthetic stream. +/// A late subscriber may start at any sequence, but it can never move backwards +/// or silently switch to another upstream response. +#[derive(Debug)] +pub struct FrameValidator { + expected_run_id: [u8; 16], + expected_marker: u32, + origin_stream_id: Option<[u8; 16]>, + last_sequence: Option, +} + +impl FrameValidator { + #[must_use] + pub fn new(run_id: &RunId, expected_marker: u32) -> Self { + let mut expected_run_id = [0_u8; 16]; + expected_run_id.copy_from_slice(&blake3::hash(run_id.0.as_bytes()).as_bytes()[..16]); + Self { expected_run_id, expected_marker, origin_stream_id: None, last_sequence: None } + } + + pub fn validate(&mut self, frame: &Frame) -> Result<(), TestkitError> { + if frame.run_id != self.expected_run_id { + return Err(TestkitError::Protocol("received a frame for a foreign test run".to_owned())); + } + if frame.channel_marker != self.expected_marker { + return Err(TestkitError::Protocol(format!( + "marker mismatch: expected {}, got {}", + self.expected_marker, frame.channel_marker + ))); + } + if let Some(origin_stream_id) = self.origin_stream_id { + if origin_stream_id != frame.origin_stream_id { + return Err(TestkitError::Protocol("origin stream changed during a playback".to_owned())); + } + } else { + self.origin_stream_id = Some(frame.origin_stream_id); + } + if let Some(previous) = self.last_sequence { + if frame.sequence <= previous { + return Err(TestkitError::Protocol(format!( + "non-increasing frame sequence: previous {previous}, current {}", + frame.sequence + ))); + } + } + let expected_payload = format!("marker={};sequence={}\n", frame.channel_marker, frame.sequence); + if frame.payload.as_ref() != expected_payload.as_bytes() { + return Err(TestkitError::Protocol( + "synthetic frame payload does not match marker and sequence".to_owned(), + )); + } + self.last_sequence = Some(frame.sequence); + Ok(()) + } +} + +impl FrameDecoder { + pub fn push(&mut self, bytes: &[u8]) -> Result, TestkitError> { + self.buffer.extend_from_slice(bytes); + let mut frames = Vec::new(); + loop { + let Some(position) = self.buffer.windows(MAGIC.len()).position(|window| window == MAGIC) else { + self.keep_magic_suffix(); + return Ok(frames); + }; + if position != 0 { + self.buffer.advance(position); + } + if self.buffer.len() < FIXED_HEADER_LEN { + return Ok(frames); + } + let header_len = u16::from_be_bytes([self.buffer[10], self.buffer[11]]) as usize; + if header_len != FIXED_HEADER_LEN || self.buffer[8] != VERSION { + self.buffer.advance(1); + continue; + } + let expected_header_crc = u32::from_be_bytes( + self.buffer[FIXED_HEADER_LEN - 4..FIXED_HEADER_LEN] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid header length".to_owned()))?, + ); + if hash(&self.buffer[..FIXED_HEADER_LEN - 4]) != expected_header_crc { + self.buffer.advance(1); + continue; + } + let payload_len = u32::from_be_bytes( + self.buffer[68..72] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid payload length".to_owned()))?, + ) as usize; + if payload_len > MAX_PAYLOAD_LEN { + self.buffer.advance(1); + continue; + } + let total_len = FIXED_HEADER_LEN + payload_len + TRAILER_LEN; + if self.buffer.len() < total_len { + return Ok(frames); + } + let payload_start = FIXED_HEADER_LEN; + let payload_end = payload_start + payload_len; + let expected_payload_crc = u32::from_be_bytes( + self.buffer[payload_end..total_len] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid trailer length".to_owned()))?, + ); + if hash(&self.buffer[payload_start..payload_end]) != expected_payload_crc { + self.buffer.advance(1); + continue; + } + let mut run_id = [0_u8; 16]; + let mut origin_stream_id = [0_u8; 16]; + run_id.copy_from_slice(&self.buffer[12..28]); + origin_stream_id.copy_from_slice(&self.buffer[28..44]); + let frame = Frame { + run_id, + origin_stream_id, + channel_marker: u32::from_be_bytes( + self.buffer[44..48].try_into().map_err(|_| TestkitError::Protocol("invalid marker".to_owned()))?, + ), + sequence: u64::from_be_bytes( + self.buffer[48..56] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid sequence".to_owned()))?, + ), + origin_mono_nanos: u64::from_be_bytes( + self.buffer[56..64] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid timestamp".to_owned()))?, + ), + configured_bps: u32::from_be_bytes( + self.buffer[64..68].try_into().map_err(|_| TestkitError::Protocol("invalid bitrate".to_owned()))?, + ), + payload: self.buffer[payload_start..payload_end].to_vec().into(), + }; + self.buffer.advance(total_len); + frames.push(frame); + } + } + + fn keep_magic_suffix(&mut self) { + let retained = self.buffer.len().min(MAGIC.len() - 1); + let start = self.buffer.len() - retained; + let suffix = self.buffer.split_off(start); + self.buffer = suffix; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_every_network_split() { + let frame = Frame::synthetic(&RunId::new("run"), &OriginStreamId::new("origin"), 17, 0); + let mut encoded = BytesMut::new(); + frame.encode(&mut encoded).unwrap(); + for split in 0..=encoded.len() { + let mut decoder = FrameDecoder::default(); + let mut frames = decoder.push(&encoded[..split]).unwrap(); + frames.extend(decoder.push(&encoded[split..]).unwrap()); + assert_eq!(frames, vec![frame.clone()]); + } + } + + #[test] + fn validator_rejects_foreign_runs_and_sequence_replay() { + let run = RunId::new("run"); + let stream = OriginStreamId::new("origin"); + let mut validator = FrameValidator::new(&run, 17); + assert!(validator.validate(&Frame::synthetic(&run, &stream, 17, 4)).is_ok()); + assert!(validator.validate(&Frame::synthetic(&run, &stream, 17, 4)).is_err()); + assert!(validator.validate(&Frame::synthetic(&RunId::new("other"), &stream, 17, 5)).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/hls.rs b/tools/tuliprox-testkit/src/hls.rs new file mode 100644 index 000000000..e15d65d43 --- /dev/null +++ b/tools/tuliprox-testkit/src/hls.rs @@ -0,0 +1,124 @@ +use crate::{ + frame::{FrameDecoder, FrameValidator}, + protocol::RunId, + TestkitError, +}; +use futures::StreamExt; +use std::collections::BTreeMap; +use url::Url; + +const SEGMENT_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); + +pub async fn read_segments( + manifest_url: &str, + expected_run_id: &RunId, + expected_marker: u32, + minimum_frames: u64, + headers: &BTreeMap, +) -> Result { + let client = reqwest::Client::builder().http1_only().build()?; + let manifest = + request_with_headers(client.get(manifest_url), headers).send().await?.error_for_status()?.text().await?; + let base = Url::parse(manifest_url).map_err(|error| TestkitError::Configuration(error.to_string()))?; + let segments = manifest.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#')); + let mut frames = 0; + let mut validator = FrameValidator::new(expected_run_id, expected_marker); + for segment in segments { + let segment_url = base.join(segment).map_err(|error| TestkitError::Configuration(error.to_string()))?; + let response = request_with_headers(client.get(segment_url), headers).send().await?.error_for_status()?; + let mut decoder = FrameDecoder::default(); + let mut body = response.bytes_stream(); + loop { + let chunk = match tokio::time::timeout(SEGMENT_IDLE_TIMEOUT, body.next()).await { + Ok(Some(chunk)) => chunk?, + Ok(None) => break, + Err(_) => return Err(TestkitError::Protocol("HLS segment idle timeout".to_owned())), + }; + for frame in decoder.push(&chunk)? { + validator.validate(&frame)?; + frames += 1; + } + } + if frames >= minimum_frames { + return Ok(frames); + } + } + Err(TestkitError::Protocol(format!("HLS produced {frames} valid frames, expected at least {minimum_frames}"))) +} + +fn request_with_headers( + mut request: reqwest::RequestBuilder, + headers: &BTreeMap, +) -> reqwest::RequestBuilder { + for (name, value) in headers { + request = request.header(name, value); + } + request +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{frame::Frame, protocol::OriginStreamId}; + use axum::{ + body::Body, + extract::OriginalUri, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::get, + Router, + }; + use bytes::BytesMut; + use std::sync::Arc; + use tokio::{net::TcpListener, sync::Mutex}; + + #[test] + fn manifest_entries_exclude_directives() { + let entries = "#EXTM3U\n#EXTINF:1,\n0.ts\n" + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .collect::>(); + assert_eq!(entries, vec!["0.ts"]); + } + + #[tokio::test] + async fn forwards_actor_headers_to_manifest_and_segments() -> Result<(), TestkitError> { + let run_id = RunId::new("hls-auth"); + let mut payload = BytesMut::new(); + Frame::synthetic(&run_id, &OriginStreamId::new("segment"), 17, 0).encode(&mut payload)?; + let payload = Arc::new(payload.freeze()); + let requests = Arc::new(Mutex::new(Vec::new())); + let app = Router::new().fallback(get({ + let payload = Arc::clone(&payload); + let requests = Arc::clone(&requests); + move |headers: HeaderMap, uri: OriginalUri| { + let payload = Arc::clone(&payload); + let requests = Arc::clone(&requests); + async move { + if headers.get("x-testkit-token").and_then(|value| value.to_str().ok()) != Some("accepted") { + return StatusCode::UNAUTHORIZED.into_response(); + } + requests.lock().await.push(uri.path().to_owned()); + match uri.path() { + "/index.m3u8" => "#EXTM3U\n#EXTINF:1,\nsegment.ts\n".into_response(), + "/segment.ts" => Response::new(Body::from(payload.as_ref().clone())), + _ => StatusCode::NOT_FOUND.into_response(), + } + } + } + })); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let headers = BTreeMap::from([(String::from("x-testkit-token"), String::from("accepted"))]); + let received = read_segments(&format!("http://{address}/index.m3u8"), &run_id, 17, 1, &headers).await?; + server.abort(); + + assert_eq!(received, 1); + assert_eq!(requests.lock().await.as_slice(), ["/index.m3u8", "/segment.ts"]); + Ok(()) + } +} diff --git a/tools/tuliprox-testkit/src/lib.rs b/tools/tuliprox-testkit/src/lib.rs new file mode 100644 index 000000000..f3adf046f --- /dev/null +++ b/tools/tuliprox-testkit/src/lib.rs @@ -0,0 +1,57 @@ +#![forbid(unsafe_code)] + +pub mod adaptive; +pub mod bootstrap; +pub mod config; +pub mod control; +pub mod custom_video; +pub mod discovery; +pub mod faults; +pub mod frame; +pub mod hls; +pub mod observation; +pub mod oracle; +pub mod origin_events; +pub mod origin_transport; +pub mod policy; +pub mod protocol; +pub mod report; +pub mod scheduler; +pub mod secret; +pub mod shared; +pub mod transport; +pub mod vod; + +use std::process::ExitCode; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TestkitError { + #[error("invalid configuration: {0}")] + Configuration(String), + #[error("protocol error: {0}")] + Protocol(String), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("HTTP error: {0}")] + Http(#[from] reqwest::Error), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunExit { + Passed, + Failed, + Invalid, + Inconclusive, +} + +impl From for ExitCode { + fn from(value: RunExit) -> Self { + match value { + RunExit::Passed => ExitCode::SUCCESS, + RunExit::Failed => ExitCode::from(1), + RunExit::Invalid => ExitCode::from(2), + RunExit::Inconclusive => ExitCode::from(3), + } + } +} diff --git a/tools/tuliprox-testkit/src/main.rs b/tools/tuliprox-testkit/src/main.rs new file mode 100644 index 000000000..38d198c97 --- /dev/null +++ b/tools/tuliprox-testkit/src/main.rs @@ -0,0 +1,3107 @@ +use axum::{ + body::Body, + extract::{Extension, Path, Query, State}, + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post, put}, + Json, Router, +}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use bytes::BytesMut; +use clap::{Parser, Subcommand}; +use futures::StreamExt; +use serde::Serialize; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + fmt::Write, + net::SocketAddr, + path::PathBuf, + process::ExitCode, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tokio::{ + net::TcpListener, + sync::{oneshot, watch, Mutex}, +}; +use tuliprox_testkit::{ + bootstrap::{fixture_password, IsolatedFixture}, + config::{ExpectedPlayback, PostReadAction, Scenario}, + control::{PlaybackRegistry, StartDisposition}, + custom_video::WireMarkerScanner, + discovery::VirtualIdMap, + faults::{FaultSchedule, OriginFault}, + frame::{Frame, FrameDecoder, FrameValidator}, + hls::read_segments, + observation::TuliproxObserver, + oracle::{AdmissionOracle, Decision as AdmissionDecision, Request as AdmissionRequest}, + origin_events::{BodyCloseReason, OriginEvent, OriginEventKind, OriginStats}, + origin_transport::{ + serve_tracked_media_listener, OriginConnectionMeta, OriginLimitMode, OriginPolicy, OriginTracker, + }, + protocol::{ + AgentId, AgentMessage, Command as ControlCommand, CommandId, Envelope, OriginStreamId, PlaybackEvent, + PlaybackId, PlaybackOutcome, RejectionReason, RunId, RunLease, + }, + report::{ReportEvent, RunReport}, + secret::environment, + transport::{controller_websocket, AgentControlConnection, ControllerState, SharedControllerState}, + RunExit, TestkitError, +}; + +fn is_rejection_http_status(status: u16, headers: &reqwest::header::HeaderMap) -> bool { + matches!(status, 403 | 429) || ((status == 502 || status == 503) && headers.get("x-tuliprox-rejection").is_some()) +} + +const PLAYBACK_IDLE_TIMEOUT: Duration = Duration::from_secs(15); +const PLAYLIST_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Parser)] +#[command(name = "tuliprox-testkit", version, about = "Tuliprox streaming admission test kit")] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Serve a controllable synthetic live-stream origin. + Origin { + #[arg(long, default_value = "127.0.0.1:9910")] + listen: SocketAddr, + #[arg(long)] + control_listen: Option, + #[arg(long, default_value = "local-run")] + run_id: String, + #[arg(long, default_value_t = 64_000)] + bitrate: u32, + #[arg(long, value_delimiter = ',', default_value = "17")] + markers: Vec, + #[arg(long)] + account_limit: Option, + #[arg(long, default_value = "observe_only")] + limit_mode: String, + }, + /// Execute a local, sequential scenario against Tuliprox URLs. + Controller { + #[arg(long)] + scenario: PathBuf, + #[arg(long, default_value = "testkit-report")] + report_directory: PathBuf, + #[arg(long, default_value = "127.0.0.1:9900")] + agent_listen: SocketAddr, + #[arg(long)] + run_id: Option, + }, + /// Read and validate one framed playback URL; useful on a remote listener host. + Agent { + #[arg(long)] + url: String, + #[arg(long)] + expected_marker: u32, + #[arg(long, default_value_t = 5)] + frames: u64, + #[arg(long)] + hls: bool, + #[arg(long)] + controller: Option, + #[arg(long, default_value = "local-agent")] + agent_id: String, + #[arg(long, default_value = "local-run")] + run_id: String, + }, +} + +#[derive(Clone)] +struct OriginState { + run_id: RunId, + bitrate: u32, + markers: Arc>, + stream_counter: Arc>, + observations: Arc>>, + faults: Arc>, + hls_sequences: Arc>>, + tracker: OriginTracker, +} + +#[derive(Clone, Serialize)] +struct OriginObservation { + origin_stream_id: String, + channel_marker: u32, + frames_emitted: u64, + bytes_emitted: u64, +} + +#[derive(Serialize)] +struct OriginInstance { + run_id: String, +} + +#[tokio::main] +async fn main() -> ExitCode { + match run(Cli::parse()).await { + Ok(exit) => exit.into(), + Err(error) => { + eprintln!("{error}"); + match error { + TestkitError::Configuration(_) => RunExit::Invalid.into(), + TestkitError::Protocol(_) | TestkitError::Io(_) | TestkitError::Http(_) => RunExit::Inconclusive.into(), + } + } + } +} + +async fn run(cli: Cli) -> Result { + match cli.command { + Command::Origin { listen, control_listen, run_id, bitrate, mut markers, account_limit, limit_mode } => { + markers.sort_unstable(); + markers.dedup(); + if markers.is_empty() { + return Err(TestkitError::Configuration("origin requires at least one channel marker".to_owned())); + } + let mode = match limit_mode.as_str() { + "reject_new" => OriginLimitMode::RejectNew, + "evict_oldest" => OriginLimitMode::EvictOldest, + _ => OriginLimitMode::ObserveOnly, + }; + let policy = OriginPolicy { account_limit, limit_mode: mode }; + let tracker = OriginTracker::new(&run_id, policy); + serve_origin( + listen, + control_listen, + OriginState { + run_id: RunId::new(run_id), + bitrate, + markers: Arc::new(markers), + stream_counter: Arc::new(Mutex::new(0)), + observations: Arc::new(Mutex::new(Vec::new())), + faults: Arc::new(Mutex::new(FaultSchedule::default())), + hls_sequences: Arc::new(Mutex::new(HashMap::new())), + tracker, + }, + ) + .await + } + Command::Agent { url, expected_marker, frames, hls, controller, agent_id, run_id } => { + if let Some(controller) = controller { + return run_controlled_agent(&controller, AgentId::new(agent_id), RunId::new(run_id)).await; + } + if hls { + match read_segments(&url, &RunId::new(run_id), expected_marker, frames, &BTreeMap::new()).await { + Ok(received) => { + println!("received {received} valid HLS frames for marker {expected_marker}"); + Ok(RunExit::Passed) + } + Err(error) => { + eprintln!("{error}"); + Ok(RunExit::Failed) + } + } + } else { + let outcome = run_agent(&url, &RunId::new(run_id), expected_marker, frames, &BTreeMap::new()).await?; + Ok(if outcome.is_streaming() { RunExit::Passed } else { RunExit::Failed }) + } + } + Command::Controller { scenario, report_directory, agent_listen, run_id } => { + run_controller(&scenario, &report_directory, agent_listen, run_id.as_deref()).await + } + } +} + +#[allow(clippy::too_many_lines)] +async fn run_controlled_agent(controller: &str, agent_id: AgentId, run_id: RunId) -> Result { + let hello = Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: "agent-hello".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Hello { + hostname: "remote-agent".to_owned(), + supported_protocols: vec!["xtream_ts".to_owned(), "hls".to_owned()], + maximum_listeners: 1, + }, + }; + let mut connection = AgentControlConnection::connect(controller, hello).await?; + connection + .send_event(Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: "agent-ready".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: 2, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Ready, + }) + .await?; + let mut sequence = 3; + let mut lease = RunLease::new(1, Duration::from_secs(15)); + let mut active: HashMap> = HashMap::new(); + let (term_tx, mut term_rx) = tokio::sync::mpsc::channel::<(PlaybackId, PlaybackOutcome)>(64); + let mut pending_stops: HashMap> = HashMap::new(); + let mut playback_registry = PlaybackRegistry::default(); + let mut heartbeats = tokio::time::interval(Duration::from_secs(5)); + heartbeats.tick().await; + loop { + let command = tokio::select! { + Some((playback_id, outcome)) = term_rx.recv() => { + active.remove(&playback_id.0); + let caused_by = pending_stops.remove(&playback_id.0).flatten(); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: outcome.to_string(), + typed_outcome: Some(outcome), + }, + ) + .await?; + continue; + } + command = connection.next_command() => command?, + _ = heartbeats.tick() => { + connection.send_event(Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: format!("heartbeat-{sequence}"), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: sequence, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Heartbeat, + }).await?; + sequence = sequence.saturating_add(1); + continue; + } + () = tokio::time::sleep_until(lease.deadline()) => { + release_active_playbacks(&mut active); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + None, + PlaybackEvent::LeaseExpired, + ).await?; + return Ok(RunExit::Inconclusive); + } + }; + let Some(command) = command else { + release_active_playbacks(&mut active); + return Ok(RunExit::Inconclusive); + }; + if lease.expired() { + release_active_playbacks(&mut active); + return Ok(RunExit::Inconclusive); + } + if command.run_id != run_id || command.run_generation != 1 { + return Err(TestkitError::Protocol("controller sent a stale or foreign command".to_owned())); + } + let caused_by = command.caused_by_command_id.clone(); + let command_id = caused_by.clone().unwrap_or_else(|| CommandId::new(command.message_id.clone())); + match command.payload { + ControlCommand::StartPlayback { + playback_id, + url, + headers, + expected_run_id, + expected_marker, + required_frames, + } => { + let fingerprint = playback_start_fingerprint(&url, &headers); + match playback_registry.register_start(run_id.clone(), 1, playback_id.clone(), fingerprint)? { + StartDisposition::Replay => { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::CommandAccepted { command_id }, + ) + .await?; + continue; + } + StartDisposition::New => {} + } + let target_run_id = expected_run_id.unwrap_or_else(|| run_id.clone()); + let marker = expected_marker.map_or_else(|| marker_from_url(&url), Ok)?; + let frames = required_frames.unwrap_or(5); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by.clone(), + PlaybackEvent::CommandAccepted { command_id }, + ) + .await?; + match start_held_playback(playback_id.0.clone(), &target_run_id, &url, marker, frames, &headers).await? + { + Ok(held) => { + let term_tx_clone = term_tx.clone(); + let p_id = playback_id.clone(); + tokio::spawn(async move { + let outcome = match held.task.await { + Ok(Ok(outcome)) => outcome, + Ok(Err(err)) => PlaybackOutcome::InfrastructureError { message: err.to_string() }, + Err(join_err) => PlaybackOutcome::InfrastructureError { message: join_err.to_string() }, + }; + let _ = term_tx_clone.send((p_id, outcome)).await; + }); + active.insert(playback_id.0.clone(), held.release); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::FirstValidFrame { playback_id, sequence: 0 }, + ) + .await?; + } + Err(outcome) => { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: outcome.to_string(), + typed_outcome: Some(outcome), + }, + ) + .await?; + } + } + } + ControlCommand::StopAll => { + release_active_playbacks(&mut active); + return Ok(RunExit::Passed); + } + ControlCommand::StopPlayback { playback_id } => { + if !playback_registry.stop(&run_id, 1, &playback_id) { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: "protocol_rejected".to_owned(), + typed_outcome: Some(PlaybackOutcome::InfrastructureError { + message: "protocol_rejected".to_owned(), + }), + }, + ) + .await?; + continue; + } + if let Some(release) = active.remove(&playback_id.0) { + pending_stops.insert(playback_id.0.clone(), caused_by); + let _ = release.send(()); + } else { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: "cancelled".to_owned(), + typed_outcome: Some(PlaybackOutcome::ExplicitStop), + }, + ) + .await?; + } + } + ControlCommand::ConfigureRun { lease_millis } | ControlCommand::RenewRunLease { lease_millis } => { + lease.renew(1, Duration::from_millis(lease_millis.max(1)))?; + } + } + } +} + +fn playback_start_fingerprint(url: &str, headers: &BTreeMap) -> String { + let mut source = String::with_capacity(url.len().saturating_add(headers.len().saturating_mul(32))); + source.push_str(url); + for (name, value) in headers { + source.push('\n'); + source.push_str(name); + source.push(':'); + source.push_str(value); + } + blake3::hash(source.as_bytes()).to_hex().to_string() +} + +fn release_active_playbacks(active: &mut HashMap>) { + for (_, release) in active.drain() { + let _ = release.send(()); + } +} + +async fn send_agent_event( + connection: &AgentControlConnection, + run_id: &RunId, + agent_id: &AgentId, + sequence: &mut u64, + caused_by_command_id: Option, + event: PlaybackEvent, +) -> Result<(), TestkitError> { + connection + .send_event(Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: format!("event-{sequence}"), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: *sequence, + caused_by_command_id, + local_elapsed_nanos: 0, + payload: AgentMessage::Event { event }, + }) + .await?; + *sequence = sequence.saturating_add(1); + Ok(()) +} + +struct BodyDropGuard { + conn_id: u64, + req_id: u64, + tracker: OriginTracker, + bytes_emitted: Arc, + start_time: Instant, + closed: Arc, + evicted: Arc, +} + +impl Drop for BodyDropGuard { + fn drop(&mut self) { + if !self.closed.swap(true, Ordering::SeqCst) { + let conn_id = self.conn_id; + let req_id = self.req_id; + let tracker = self.tracker.clone(); + let bytes = self.bytes_emitted.load(Ordering::Relaxed); + let duration_ms = u64::try_from(self.start_time.elapsed().as_millis()).unwrap_or(u64::MAX); + let reason = if self.evicted.load(Ordering::Relaxed) { + BodyCloseReason::Evicted + } else { + BodyCloseReason::ClientDisconnected + }; + tokio::spawn(async move { + tracker.on_body_closed(conn_id, req_id, bytes, duration_ms, reason).await; + }); + } + } +} + +async fn serve_origin( + media_listen: SocketAddr, + control_listen: Option, + state: OriginState, +) -> Result { + let media_listener = TcpListener::bind(media_listen).await?; + let media_app = Router::new() + .route("/catalog/input.m3u", get(catalog)) + .route("/live/{*stream}", get(live)) + .route("/hls/{*resource}", get(hls)) + .route("/vod/{*object}", get(vod).head(vod_head)) + .with_state(state.clone()); + + let control_app = Router::new() + .route("/health", get(|| async { StatusCode::NO_CONTENT })) + .route("/v1/instance", get(origin_instance)) + .route("/v1/runs/{run_id}/reset", post(reset_origin_run)) + .route("/v1/runs/{run_id}/connections", get(origin_connections)) + .route("/v1/runs/{run_id}/events", get(origin_events)) + .route("/v1/runs/{run_id}/stats", get(origin_stats)) + .route("/v1/runs/{run_id}/faults/{fault_id}", put(set_fault).delete(clear_fault)) + .with_state(state.clone()); + + let control_addr = control_listen.unwrap_or_else(|| SocketAddr::from(([127, 0, 0, 1], 0))); + let control_listener = TcpListener::bind(control_addr).await?; + let control_task = tokio::spawn(async move { + let _ = axum::serve(control_listener, control_app).await; + }); + + let tracker = state.tracker.clone(); + let media_res = serve_tracked_media_listener(media_listener, media_app, tracker).await; + + control_task.abort(); + media_res.map_err(TestkitError::Io)?; + Ok(RunExit::Passed) +} + +fn catalog_m3u(state: &OriginState, host: &str) -> String { + let mut catalog = String::from("#EXTM3U\n"); + for marker in state.markers.iter() { + let _ = writeln!(catalog, "#EXTINF:-1 tvg-id=\"test-{marker}\",Test channel {marker}"); + let _ = writeln!(catalog, "http://{host}/live/{marker}.ts?run={}", state.run_id.0); + } + let _ = writeln!(catalog, "#EXTINF:-1 tvg-id=\"test-vod-movie.mkv\" tvg-type=\"movie\",Test Movie"); + let _ = writeln!(catalog, "http://{host}/vod/movie.mkv?run={}", state.run_id.0); + catalog +} + +async fn catalog(State(state): State, headers: HeaderMap) -> impl IntoResponse { + let host = headers.get(header::HOST).and_then(|value| value.to_str().ok()).unwrap_or("127.0.0.1"); + ([(header::CONNECTION, "close")], catalog_m3u(&state, host)) +} + +struct LiveStreamState { + ticker: tokio::time::Interval, + evict_rx: oneshot::Receiver<()>, + evicted_flag: Arc, + run_id: RunId, + stream_id: OriginStreamId, + marker: u32, + index: u64, + observations: Arc>>, + observed_stream_id: String, + bytes_emitted: Arc, + _guard: Arc, +} + +#[allow(clippy::too_many_lines)] +async fn live( + Path(stream): Path, + Query(query): Query>, + State(state): State, + meta: Option>, + headers: HeaderMap, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let Ok(marker) = stream.strip_suffix(".ts").unwrap_or(&stream).parse::() else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !state.markers.contains(&marker) { + return StatusCode::NOT_FOUND.into_response(); + } + if matches!(state.faults.lock().await.take("close-before-first-byte"), Some(OriginFault::CloseBeforeFirstByte)) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + + let conn_id = meta.as_ref().map_or(0, |m| m.conn_id); + let range_str = headers.get(header::RANGE).and_then(|v| v.to_str().ok()); + let ua = headers.get(header::USER_AGENT).and_then(|v| v.to_str().ok()); + let req_res = state + .tracker + .on_request_started( + conn_id, + "GET", + &format!("/live/{stream}"), + range_str, + ua, + query.get("account").map(String::as_str), + ) + .await; + + let req_id = match req_res { + Ok(id) => id, + Err((_id, status)) => { + return StatusCode::from_u16(status).unwrap_or(StatusCode::TOO_MANY_REQUESTS).into_response() + } + }; + + let mut counter = state.stream_counter.lock().await; + *counter += 1; + let stream_id = OriginStreamId::new(format!("origin-{}", *counter)); + drop(counter); + state.observations.lock().await.push(OriginObservation { + origin_stream_id: stream_id.0.clone(), + channel_marker: marker, + frames_emitted: 0, + bytes_emitted: 0, + }); + + let evict_rx = state.tracker.on_body_started(conn_id, req_id, None).await; + + let run_id = state.run_id.clone(); + let bitrate = state.bitrate; + let mut sample = BytesMut::new(); + if Frame::synthetic(&run_id, &stream_id, marker, 0).encode(&mut sample).is_err() { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + let bits = u128::try_from(sample.len()).unwrap_or(u128::MAX).saturating_mul(8); + let nanos = bits.saturating_mul(1_000_000_000).checked_div(u128::from(bitrate.max(1))).unwrap_or(1); + let interval = Duration::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX).max(1)); + let observations = state.observations.clone(); + let observed_stream_id = stream_id.0.clone(); + + let bytes_emitted = Arc::new(AtomicU64::new(0)); + let closed = Arc::new(AtomicBool::new(false)); + let evicted = Arc::new(AtomicBool::new(false)); + let guard = Arc::new(BodyDropGuard { + conn_id, + req_id, + tracker: state.tracker.clone(), + bytes_emitted: bytes_emitted.clone(), + start_time: Instant::now(), + closed: closed.clone(), + evicted: evicted.clone(), + }); + + let evicted_flag = evicted.clone(); + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + let stream_state = LiveStreamState { + ticker, + evict_rx, + evicted_flag, + run_id, + stream_id, + marker, + index: 0, + observations, + observed_stream_id, + bytes_emitted, + _guard: guard, + }; + + let body = futures::stream::unfold(stream_state, |mut state| async move { + tokio::select! { + biased; + res = &mut state.evict_rx => { + if res.is_ok() { + state.evicted_flag.store(true, Ordering::Release); + } + None + } + _ = state.ticker.tick() => { + let frame = Frame::synthetic(&state.run_id, &state.stream_id, state.marker, state.index); + state.index = state.index.saturating_add(1); + let mut encoded = BytesMut::new(); + if frame.encode(&mut encoded).is_err() { + return None; + } + let bytes = encoded.len() as u64; + state.bytes_emitted.fetch_add(bytes, Ordering::Relaxed); + if let Some(observation) = state.observations + .lock() + .await + .iter_mut() + .find(|observation| observation.origin_stream_id == state.observed_stream_id) + { + observation.frames_emitted = observation.frames_emitted.saturating_add(1); + observation.bytes_emitted = observation.bytes_emitted.saturating_add(bytes); + } + Some((Ok::<_, std::convert::Infallible>(encoded.freeze()), state)) + } + } + }); + let mut response = Response::new(Body::from_stream(body)); + response.headers_mut().insert(header::CONTENT_TYPE, HeaderValue::from_static("video/mp2t")); + if let Ok(value) = HeaderValue::from_str(&bitrate.to_string()) { + response.headers_mut().insert("x-tpx-test-bitrate", value); + } + response +} + +async fn origin_connections(Path(run_id): Path, State(state): State) -> Response { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND.into_response(); + } + Json(state.observations.lock().await.clone()).into_response() +} + +async fn origin_events(Path(run_id): Path, State(state): State) -> Response { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND.into_response(); + } + Json(state.tracker.events().await).into_response() +} + +async fn origin_stats(Path(run_id): Path, State(state): State) -> Response { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND.into_response(); + } + Json(state.tracker.stats().await).into_response() +} + +async fn origin_instance(State(state): State) -> Json { + Json(OriginInstance { run_id: state.run_id.0.clone() }) +} + +async fn reset_origin_run(Path(run_id): Path, State(state): State) -> StatusCode { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND; + } + *state.stream_counter.lock().await = 0; + state.hls_sequences.lock().await.clear(); + state.observations.lock().await.clear(); + state.tracker.reset().await; + StatusCode::NO_CONTENT +} + +async fn set_fault(Path((run_id, fault_id)): Path<(String, String)>, State(state): State) -> StatusCode { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND; + } + let fault = match fault_id.as_str() { + "close-before-first-byte" => OriginFault::CloseBeforeFirstByte, + _ => return StatusCode::BAD_REQUEST, + }; + match state.faults.lock().await.insert(fault_id, fault) { + Ok(()) => StatusCode::NO_CONTENT, + Err(_) => StatusCode::CONFLICT, + } +} + +async fn clear_fault(Path((run_id, fault_id)): Path<(String, String)>, State(state): State) -> StatusCode { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND; + } + match state.faults.lock().await.clear(&fault_id) { + Ok(()) => StatusCode::NO_CONTENT, + Err(_) => StatusCode::NOT_FOUND, + } +} + +async fn hls( + Path(resource): Path, + Query(query): Query>, + State(state): State, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let Some((marker, resource)) = resource.split_once('/') else { + return StatusCode::NOT_FOUND.into_response(); + }; + let Ok(marker) = marker.parse::() else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !state.markers.contains(&marker) { + return StatusCode::NOT_FOUND.into_response(); + } + if resource == "index.m3u8" { + let mut sequences = state.hls_sequences.lock().await; + let sequence = sequences.entry(marker).or_insert(0); + *sequence += 1; + let first = sequence.saturating_sub(1); + let playlist = format!( + "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:1\n#EXT-X-MEDIA-SEQUENCE:{first}\n#EXTINF:1.0,\n{first}.ts?run={}\n#EXTINF:1.0,\n{}.ts?run={}\n", + state.run_id.0, *sequence, state.run_id.0 + ); + return ([(header::CONTENT_TYPE, "application/vnd.apple.mpegurl")], playlist).into_response(); + } + let segment = resource.strip_suffix(".ts").and_then(|part| part.parse::().ok()); + let Some(sequence) = segment else { + return StatusCode::NOT_FOUND.into_response(); + }; + let frame = + Frame::synthetic(&state.run_id, &OriginStreamId::new(format!("hls-presentation-{marker}")), marker, sequence); + let mut encoded = BytesMut::new(); + if frame.encode(&mut encoded).is_err() { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + ([(header::CONTENT_TYPE, "video/mp2t")], encoded.freeze()).into_response() +} + +async fn vod( + Path(object): Path, + Query(query): Query>, + State(state): State, + meta: Option>, + headers: HeaderMap, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let conn_id = meta.as_ref().map_or(0, |m| m.conn_id); + let total_size = + query.get("size").and_then(|s| s.parse::().ok()).unwrap_or(tuliprox_testkit::vod::DEFAULT_VOD_SIZE); + + let range_header = headers.get(header::RANGE).and_then(|value| value.to_str().ok()); + let ua = headers.get(header::USER_AGENT).and_then(|value| value.to_str().ok()); + let req_res = state + .tracker + .on_request_started( + conn_id, + "GET", + &format!("/vod/{object}"), + range_header, + ua, + query.get("account").map(String::as_str), + ) + .await; + + let req_id = match req_res { + Ok(id) => id, + Err((_id, status)) => { + return StatusCode::from_u16(status).unwrap_or(StatusCode::TOO_MANY_REQUESTS).into_response() + } + }; + + let (start, end, status) = match range_header { + None => (0, total_size, StatusCode::OK), + Some(range_str) => { + if let Some((start, end)) = tuliprox_testkit::vod::parse_range_header(range_str, total_size) { + (start, end, StatusCode::PARTIAL_CONTENT) + } else { + let mut response = StatusCode::RANGE_NOT_SATISFIABLE.into_response(); + if let Ok(value) = HeaderValue::from_str(&format!("bytes */{total_size}")) { + response.headers_mut().insert(header::CONTENT_RANGE, value); + } + apply_vod_validators(&mut response, &vod_payload(&object)); + return response; + } + } + }; + + let length = end.saturating_sub(start); + let evict_rx = state.tracker.on_body_started(conn_id, req_id, Some(length)).await; + let stall_ms = query.get("stall_ms").and_then(|s| s.parse::().ok()); + let abort_after_bytes = query.get("abort_after_bytes").and_then(|s| s.parse::().ok()); + + let options = tuliprox_testkit::vod::VodStreamOptions { + object: object.clone(), + start_offset: start, + length, + bitrate: state.bitrate, + chunk_size: 64 * 1024, + stall_ms, + abort_after_bytes, + }; + + let bytes_emitted = Arc::new(AtomicU64::new(0)); + let closed = Arc::new(AtomicBool::new(false)); + let evicted = Arc::new(AtomicBool::new(false)); + let guard = Arc::new(BodyDropGuard { + conn_id, + req_id, + tracker: state.tracker.clone(), + bytes_emitted: bytes_emitted.clone(), + start_time: Instant::now(), + closed: closed.clone(), + evicted: evicted.clone(), + }); + let body_stream = tuliprox_testkit::vod::create_vod_stream_with_evicted(options, evict_rx, Some(evicted)); + + let bytes_tracker = bytes_emitted.clone(); + let tracked_stream = body_stream.map(move |item| { + let _keep_guard = &guard; + if let Ok(chunk) = &item { + bytes_tracker.fetch_add(chunk.len() as u64, Ordering::Relaxed); + } + item + }); + + let mut response = Response::new(Body::from_stream(tracked_stream)); + *response.status_mut() = status; + response.headers_mut().insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + if status == StatusCode::PARTIAL_CONTENT { + let value = format!("bytes {}-{}/{}", start, end.saturating_sub(1), total_size); + if let Ok(value) = HeaderValue::from_str(&value) { + response.headers_mut().insert(header::CONTENT_RANGE, value); + } + } + if let Ok(value) = HeaderValue::from_str(&length.to_string()) { + response.headers_mut().insert(header::CONTENT_LENGTH, value); + } + apply_vod_validators(&mut response, &vod_payload(&object)); + response +} + +async fn vod_head( + Path(object): Path, + Query(query): Query>, + State(state): State, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let total_size = + query.get("size").and_then(|s| s.parse::().ok()).unwrap_or(tuliprox_testkit::vod::DEFAULT_VOD_SIZE); + let mut response = StatusCode::OK.into_response(); + response.headers_mut().insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + if let Ok(value) = HeaderValue::from_str(&total_size.to_string()) { + response.headers_mut().insert(header::CONTENT_LENGTH, value); + } + apply_vod_validators(&mut response, &vod_payload(&object)); + response +} + +fn vod_payload(object: &str) -> Vec { + let digest = blake3::hash(object.as_bytes()); + digest.as_bytes().iter().copied().cycle().take(16 * 1024).collect() +} + +fn apply_vod_validators(response: &mut Response, payload: &[u8]) { + let etag = format!("\"{}\"", blake3::hash(payload).to_hex()); + if let Ok(etag) = HeaderValue::from_str(&etag) { + response.headers_mut().insert(header::ETAG, etag); + } + response.headers_mut().insert(header::LAST_MODIFIED, HeaderValue::from_static("Thu, 01 Jan 1970 00:00:00 GMT")); +} + +async fn run_agent( + url: &str, + expected_run_id: &RunId, + expected_marker: u32, + required_frames: u64, + headers: &BTreeMap, +) -> Result { + let client = reqwest::Client::builder().http1_only().pool_max_idle_per_host(0).build()?; + let mut request = client.get(url); + for (name, value) in headers { + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header name {name}: {error}")) + })?; + let value = reqwest::header::HeaderValue::from_str(value).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header value for {name}: {error}")) + })?; + request = request.header(name, value); + } + let response = match request.send().await { + Ok(res) => res, + Err(err) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + }; + let status = response.status().as_u16(); + if !response.status().is_success() { + eprintln!("playback returned {status}"); + return Ok(if is_rejection_http_status(status, response.headers()) { + PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(status) } + } else { + PlaybackOutcome::HttpError { status } + }); + } + let mut body = response.bytes_stream(); + let mut decoder = FrameDecoder::default(); + let mut validator = FrameValidator::new(expected_run_id, expected_marker); + let mut custom_video = WireMarkerScanner::default(); + let mut received_frames = 0; + let mut received_bytes = 0; + loop { + let chunk = match tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()).await { + Ok(Some(Ok(chunk))) => chunk, + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => break, + Err(_) => { + eprintln!("stream did not produce bytes before the idle deadline"); + return Ok(PlaybackOutcome::IdleTimeout); + } + }; + received_bytes += chunk.len() as u64; + if let Some(kind) = custom_video.push(&chunk)? { + eprintln!("received custom-video fallback: {kind:?}"); + if matches!( + kind, + tuliprox_testkit::custom_video::CustomVideoKind::UserConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::ProviderConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::LowPriorityPreempted + | tuliprox_testkit::custom_video::CustomVideoKind::UserAccountExpired + ) { + return Ok(PlaybackOutcome::AdmissionRejected { reason: RejectionReason::CustomVideo(kind) }); + } + return Ok(PlaybackOutcome::InvalidData { + message: format!("unexpected non-admission custom video: {kind:?}"), + }); + } + for frame in decoder.push(&chunk)? { + if let Err(error) = validator.validate(&frame) { + eprintln!("{error}"); + return Ok(PlaybackOutcome::InvalidData { message: error.to_string() }); + } + received_frames += 1; + if received_frames >= required_frames { + println!("received {received_frames} valid frames for marker {expected_marker}"); + return Ok(PlaybackOutcome::Streaming { frames: received_frames, bytes: received_bytes }); + } + } + } + eprintln!("stream ended after {received_frames} valid frames"); + Ok(PlaybackOutcome::UnexpectedEof { frames: received_frames, bytes: received_bytes }) +} + +/// Keep the HTTP body consumed after proving its frame identity. Retaining the +/// request alone is not sufficient: a bounded proxy buffer could otherwise make +/// the upstream session disappear before the following admission request. +async fn run_agent_until_released( + url: String, + expected_run_id: RunId, + expected_marker: u32, + required_frames: u64, + headers: BTreeMap, + ready: oneshot::Sender<()>, + mut release: oneshot::Receiver<()>, +) -> Result { + let client = reqwest::Client::builder().http1_only().pool_max_idle_per_host(0).build()?; + let mut request = client.get(url); + for (name, value) in &headers { + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header name {name}: {error}")) + })?; + let value = reqwest::header::HeaderValue::from_str(value).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header value for {name}: {error}")) + })?; + request = request.header(name, value); + } + let response = match request.send().await { + Ok(res) => res, + Err(err) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + }; + let status = response.status().as_u16(); + if !response.status().is_success() { + return Ok(if is_rejection_http_status(status, response.headers()) { + PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(status) } + } else { + PlaybackOutcome::HttpError { status } + }); + } + let mut body = response.bytes_stream(); + let mut decoder = FrameDecoder::default(); + let mut validator = FrameValidator::new(&expected_run_id, expected_marker); + let mut custom_video = WireMarkerScanner::default(); + let mut received_frames = 0; + let mut received_bytes = 0; + let mut ready = Some(ready); + loop { + tokio::select! { + _ = &mut release => { + return Ok(if ready.is_none() { + PlaybackOutcome::ExplicitStop + } else { + PlaybackOutcome::UnexpectedEof { frames: received_frames, bytes: received_bytes } + }); + } + chunk = tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()) => { + let chunk = match chunk { + Ok(Some(Ok(chunk))) => chunk, + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => return Ok(PlaybackOutcome::UnexpectedEof { frames: received_frames, bytes: received_bytes }), + Err(_) => return Ok(PlaybackOutcome::IdleTimeout), + }; + received_bytes += chunk.len() as u64; + if let Some(kind) = custom_video.push(&chunk)? { + if matches!( + kind, + tuliprox_testkit::custom_video::CustomVideoKind::UserConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::ProviderConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::LowPriorityPreempted + | tuliprox_testkit::custom_video::CustomVideoKind::UserAccountExpired + ) { + return Ok(PlaybackOutcome::AdmissionRejected { + reason: RejectionReason::CustomVideo(kind), + }); + } + return Ok(PlaybackOutcome::InvalidData { + message: format!("unexpected non-admission custom video: {kind:?}"), + }); + } + for frame in decoder.push(&chunk)? { + if let Err(error) = validator.validate(&frame) { + return Ok(PlaybackOutcome::InvalidData { message: error.to_string() }); + } + received_frames += 1; + if received_frames >= required_frames { + if let Some(ready) = ready.take() { + let _ = ready.send(()); + } + } + } + } + } + } +} + +async fn run_hls_until_released( + url: String, + expected_run_id: RunId, + expected_marker: u32, + required_frames: u64, + headers: BTreeMap, + ready: oneshot::Sender<()>, + mut release: oneshot::Receiver<()>, +) -> Result { + let mut ready = Some(ready); + loop { + tokio::select! { + _ = &mut release => { + return Ok(if ready.is_none() { + PlaybackOutcome::ExplicitStop + } else { + PlaybackOutcome::UnexpectedEof { frames: 0, bytes: 0 } + }); + } + result = read_segments(&url, &expected_run_id, expected_marker, required_frames, &headers) => { + if let Err(err) = result { + return Ok(PlaybackOutcome::TransportError { message: err.to_string() }); + } + if let Some(ready) = ready.take() { + let _ = ready.send(()); + } + } + } + tokio::select! { + _ = &mut release => return Ok(PlaybackOutcome::ExplicitStop), + () = tokio::time::sleep(Duration::from_millis(100)) => {} + } + } +} + +fn is_hls_manifest_url(url: &str) -> bool { + url::Url::parse(url).is_ok_and(|parsed| { + std::path::Path::new(parsed.path()).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("m3u8")) + }) +} + +struct HeldPlayback { + playback_id: String, + release: oneshot::Sender<()>, + task: tokio::task::JoinHandle>, +} + +#[derive(Clone)] +pub struct OriginObserver { + control_base_url: String, + client: reqwest::Client, +} + +impl OriginObserver { + #[must_use] + pub fn new(control_base_url: &str) -> Self { + Self { control_base_url: control_base_url.trim_end_matches('/').to_owned(), client: reqwest::Client::new() } + } + + pub async fn stats(&self, run_id: &str) -> Result { + let url = format!("{}/v1/runs/{run_id}/stats", self.control_base_url); + let resp = self.client.get(&url).send().await?; + if !resp.status().is_success() { + return Err(TestkitError::Protocol(format!("origin stats error: {}", resp.status()))); + } + let stats = resp.json::().await?; + Ok(stats) + } + + pub async fn events(&self, run_id: &str) -> Result, TestkitError> { + let url = format!("{}/v1/runs/{run_id}/events", self.control_base_url); + let resp = self.client.get(&url).send().await?; + if !resp.status().is_success() { + return Err(TestkitError::Protocol(format!("origin events error: {}", resp.status()))); + } + let events = resp.json::>().await?; + Ok(events) + } + + pub async fn reset(&self, run_id: &str) -> Result<(), TestkitError> { + let url = format!("{}/v1/runs/{run_id}/reset", self.control_base_url); + let resp = self.client.post(&url).send().await?; + if !resp.status().is_success() { + return Err(TestkitError::Protocol(format!("origin reset error: {}", resp.status()))); + } + Ok(()) + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn run_vod_until_released( + url: String, + vod_object: Option, + method: Option, + range: Option, + expected_status: Option, + expected_content_range: Option, + read_limit_bytes: Option, + post_read_action: PostReadAction, + headers: BTreeMap, + ready: oneshot::Sender<()>, + mut release: oneshot::Receiver<()>, +) -> Result { + let client = reqwest::Client::builder().http1_only().pool_max_idle_per_host(0).build()?; + let method = match method.as_deref() { + Some("HEAD") => reqwest::Method::HEAD, + _ => reqwest::Method::GET, + }; + let mut request = client.request(method.clone(), &url); + for (name, value) in &headers { + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header name {name}: {error}")) + })?; + let value = reqwest::header::HeaderValue::from_str(value).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header value for {name}: {error}")) + })?; + request = request.header(name, value); + } + if let Some(ref range_str) = range { + let name = reqwest::header::RANGE; + let value = reqwest::header::HeaderValue::from_str(range_str) + .map_err(|error| TestkitError::Configuration(format!("invalid range header value {range_str}: {error}")))?; + request = request.header(name, value); + } + let response = match request.send().await { + Ok(res) => res, + Err(err) => { + return Ok(PlaybackOutcome::TransportError { message: err.to_string() }); + } + }; + let status = response.status().as_u16(); + if is_rejection_http_status(status, response.headers()) { + return Ok(PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(status) }); + } + let expected = expected_status.unwrap_or(if range.is_some() { 206 } else { 200 }); + if status != expected { + return Ok(PlaybackOutcome::HttpError { status }); + } + let start_offset = if range.is_some() || status == 206 { + let Some(cr_header) = response.headers().get(reqwest::header::CONTENT_RANGE).and_then(|v| v.to_str().ok()) + else { + return Ok(PlaybackOutcome::InvalidData { message: "range response omits Content-Range".to_owned() }); + }; + let Some((cr_start, cr_end, cr_total)) = tuliprox_testkit::vod::parse_content_range(cr_header) else { + return Ok(PlaybackOutcome::InvalidData { message: format!("invalid Content-Range header: {cr_header}") }); + }; + if cr_start > cr_end { + return Ok(PlaybackOutcome::InvalidData { + message: format!("Content-Range start exceeds end: {cr_header}"), + }); + } + if let Some(ref exp_cr) = expected_content_range { + if cr_header != exp_cr { + return Ok(PlaybackOutcome::InvalidData { + message: format!("unexpected content-range: {cr_header:?}, expected: {exp_cr}"), + }); + } + } + if let Some(ref r) = range { + let total_for_range = cr_total.unwrap_or(tuliprox_testkit::vod::DEFAULT_VOD_SIZE); + let Some((req_start, req_end_exclusive)) = tuliprox_testkit::vod::parse_range_header(r, total_for_range) + else { + return Ok(PlaybackOutcome::InvalidData { + message: format!("failed to parse requested range header against size {total_for_range}: {r}"), + }); + }; + let req_end_inclusive = req_end_exclusive.saturating_sub(1); + if cr_start != req_start || cr_end != req_end_inclusive { + return Ok(PlaybackOutcome::InvalidData { + message: format!( + "Content-Range {cr_header} does not match requested range {req_start}-{req_end_inclusive}" + ), + }); + } + if let Some(total) = cr_total { + if cr_end >= total { + return Ok(PlaybackOutcome::InvalidData { + message: format!("Content-Range end {cr_end} exceeds or equals total size {total}"), + }); + } + } + } + let expected_len = cr_end.saturating_sub(cr_start).saturating_add(1); + if let Some(cl) = response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + { + if cl != expected_len { + return Ok(PlaybackOutcome::InvalidData { + message: format!("Content-Length {cl} does not match Content-Range length {expected_len}"), + }); + } + } + cr_start + } else { + 0 + }; + if method == reqwest::Method::HEAD { + let _ = ready.send(()); + let _ = release.await; + return Ok(PlaybackOutcome::ExplicitStop); + } + + let object_name = vod_object.as_deref().or_else(|| url.rsplit('/').next()).unwrap_or("movie.mkv"); + + let mut body = response.bytes_stream(); + let mut received_bytes: u64 = 0; + let limit = read_limit_bytes.unwrap_or(u64::MAX); + let mut ready = Some(ready); + + while received_bytes < limit { + tokio::select! { + _ = &mut release => { + return Ok(if ready.is_none() { + PlaybackOutcome::ExplicitStop + } else { + PlaybackOutcome::UnexpectedEof { frames: 0, bytes: received_bytes } + }); + } + chunk = tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()) => { + match chunk { + Ok(Some(Ok(bytes))) => { + let to_take = usize::try_from(limit.saturating_sub(received_bytes)) + .unwrap_or(usize::MAX) + .min(bytes.len()); + let chunk_offset = start_offset.saturating_add(received_bytes); + if !tuliprox_testkit::vod::verify_deterministic_slice(object_name, chunk_offset, &bytes[..to_take]) { + return Ok(PlaybackOutcome::InvalidData { + message: format!("deterministic slice verification failed for {object_name} at offset {chunk_offset}"), + }); + } + received_bytes += to_take as u64; + if received_bytes >= limit || (read_limit_bytes.is_none() && to_take == bytes.len()) { + if let Some(r) = ready.take() { + let _ = r.send(()); + } + } + } + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => { + if let Some(r) = ready.take() { + let _ = r.send(()); + } + return Ok(PlaybackOutcome::UnexpectedEof { frames: 0, bytes: received_bytes }); + } + Err(_) => return Ok(PlaybackOutcome::IdleTimeout), + } + } + } + } + + if let Some(r) = ready.take() { + let _ = r.send(()); + } + + match post_read_action { + PostReadAction::Close => { + drop(body); + Ok(PlaybackOutcome::Streaming { frames: 0, bytes: received_bytes }) + } + PostReadAction::Pause => { + let _ = release.await; + Ok(PlaybackOutcome::ExplicitStop) + } + PostReadAction::KeepOpen => loop { + tokio::select! { + _ = &mut release => return Ok(PlaybackOutcome::ExplicitStop), + chunk = tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()) => { + match chunk { + Ok(Some(Ok(bytes))) => { + let _ = bytes; + } + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => return Ok(PlaybackOutcome::ExplicitStop), + Err(_) => return Ok(PlaybackOutcome::IdleTimeout), + } + } + } + }, + } +} + +#[allow(clippy::too_many_arguments)] +async fn start_held_vod_playback( + playback_id: String, + url: &str, + vod_object: Option, + method: Option, + range: Option, + expected_status: Option, + expected_content_range: Option, + read_limit_bytes: Option, + post_read_action: PostReadAction, + headers: &BTreeMap, +) -> Result, TestkitError> { + let (ready_tx, ready_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let is_close = post_read_action == PostReadAction::Close; + let task = tokio::spawn(run_vod_until_released( + url.to_owned(), + vod_object, + method, + range, + expected_status, + expected_content_range, + read_limit_bytes, + post_read_action, + headers.clone(), + ready_tx, + release_rx, + )); + + match tokio::time::timeout(Duration::from_secs(10), ready_rx).await { + Ok(Ok(())) => { + if is_close { + match task.await { + Ok(Ok(outcome)) => Ok(Err(outcome)), + Ok(Err(err)) => Err(err), + Err(err) => Err(TestkitError::Protocol(format!("task error: {err}"))), + } + } else { + Ok(Ok(HeldPlayback { playback_id, release: release_tx, task })) + } + } + Ok(Err(_)) => match task.await { + Ok(Ok(outcome)) => Ok(Err(outcome)), + Ok(Err(error)) => Err(error), + Err(error) => Err(TestkitError::Protocol(format!("playback task failed: {error}"))), + }, + Err(_) => { + task.abort(); + let _ = task.await; + Ok(Err(PlaybackOutcome::IdleTimeout)) + } + } +} + +async fn start_held_playback( + playback_id: String, + expected_run_id: &RunId, + url: &str, + marker: u32, + frames: u64, + headers: &BTreeMap, +) -> Result, TestkitError> { + let (ready_tx, ready_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let task = if is_hls_manifest_url(url) { + tokio::spawn(run_hls_until_released( + url.to_owned(), + expected_run_id.clone(), + marker, + frames, + headers.clone(), + ready_tx, + release_rx, + )) + } else { + tokio::spawn(run_agent_until_released( + url.to_owned(), + expected_run_id.clone(), + marker, + frames, + headers.clone(), + ready_tx, + release_rx, + )) + }; + match tokio::time::timeout(Duration::from_secs(10), ready_rx).await { + Ok(Ok(())) => Ok(Ok(HeldPlayback { playback_id, release: release_tx, task })), + Ok(Err(_)) => match task.await { + Ok(Ok(outcome)) => Ok(Err(outcome)), + Ok(Err(error)) => Err(error), + Err(error) => Err(TestkitError::Protocol(format!("playback task failed: {error}"))), + }, + Err(_) => { + task.abort(); + let _ = task.await; + Ok(Err(PlaybackOutcome::IdleTimeout)) + } + } +} + +#[allow(clippy::too_many_lines)] +async fn run_controller( + path: &std::path::Path, + report_directory: &std::path::Path, + agent_listen: SocketAddr, + run_id: Option<&str>, +) -> Result { + let scenario_name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown").to_owned(); + let mut scenario = match Scenario::from_path(path) { + Ok(scenario) => scenario, + Err(err) => { + let _ = write_raw_report( + &scenario_name, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + let working_directory = path.parent().unwrap_or_else(|| std::path::Path::new(".")); + let expanded_steps = match scenario.expanded_steps() { + Ok(steps) => steps, + Err(err) => { + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + let fixture = match scenario.tuliprox.execution_mode { + tuliprox_testkit::config::ExecutionMode::IsolatedFixture => { + let Some(bootstrap) = scenario.tuliprox.bootstrap.as_ref() else { + let err = TestkitError::Configuration("isolated_fixture requires a bootstrap definition".to_owned()); + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + }; + let testkit_binary = match std::env::current_exe() { + Ok(b) => b, + Err(err) => { + let err = TestkitError::Io(err); + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("bootstrap_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + let fixture = match IsolatedFixture::start( + bootstrap, + working_directory, + &testkit_binary, + &scenario.name, + scenario.origin.as_ref(), + scenario.policy_contract.as_ref(), + &scenario.channels, + ) + .await + { + Ok(f) => f, + Err(err) => { + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("bootstrap_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + scenario.tuliprox.base_url.clone_from(&fixture.base_url); + scenario.tuliprox.api_base_url = Some(format!("{}/api/v1", fixture.base_url)); + let has_vod = + expanded_steps.iter().any(|s| s.start.as_ref().and_then(|st| st.vod_object.as_ref()).is_some()); + if !scenario.channels.is_empty() || has_vod { + let Some(username) = scenario.actors.iter().find_map(|actor| actor.username.as_deref()) else { + let err = + TestkitError::Configuration("channel or vod scenarios require an actor username".to_owned()); + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + let _ = fixture.stop().await; + return Err(err); + }; + scenario.tuliprox.playlist_url = Some(format!( + "{}/m3u?username={}&password={}", + fixture.base_url, + url::form_urlencoded::byte_serialize(username.as_bytes()).collect::(), + fixture_password(), + )); + } + Some(fixture) + } + tuliprox_testkit::config::ExecutionMode::ExistingInstance => None, + }; + let origin_control_url = fixture.as_ref().map(|f| f.origin_control_url.clone()); + let scenario_name_for_err = scenario.name.clone(); + let result = + run_controller_scenario(scenario, report_directory, agent_listen, origin_control_url.as_deref(), run_id).await; + let cleanup = match fixture { + Some(fixture) => fixture.stop().await, + None => Ok(()), + }; + match (result, cleanup) { + (Ok(outcome), Ok(())) => Ok(outcome), + (Ok(_), Err(error)) => { + let _ = write_raw_report( + &scenario_name_for_err, + report_directory, + RunExit::Failed, + Vec::new(), + "cleanup", + run_id, + Some("cleanup_error"), + Some(&error.to_string()), + Some(4), + ); + Err(error) + } + (Err(error), _) => { + let _ = write_raw_report( + &scenario_name_for_err, + report_directory, + RunExit::Failed, + Vec::new(), + "execution_error", + run_id, + Some("execution_error"), + Some(&error.to_string()), + Some(4), + ); + Err(error) + } + } +} + +#[allow(clippy::too_many_lines)] +async fn run_controller_scenario( + scenario: Scenario, + report_directory: &std::path::Path, + agent_listen: SocketAddr, + origin_control_url: Option<&str>, + run_id: Option<&str>, +) -> Result { + if scenario.tuliprox.execution_mode == tuliprox_testkit::config::ExecutionMode::ExistingInstance { + eprintln!("existing-instance mode is intentionally inconclusive until loaded-policy evidence is available"); + write_report( + &scenario, + report_directory, + RunExit::Inconclusive, + Vec::new(), + "unproven-existing-instance-policy", + run_id, + Some("execution_mode_unsupported"), + Some("existing-instance mode is intentionally inconclusive until loaded-policy evidence is available"), + Some(3), + )?; + return Ok(RunExit::Inconclusive); + } + let api_base_url = scenario + .tuliprox + .api_base_url + .clone() + .unwrap_or_else(|| format!("{}/api/v1", scenario.tuliprox.base_url.trim_end_matches('/'))); + let credentials = match (&scenario.tuliprox.admin_username, &scenario.tuliprox.admin_password_env) { + (Some(username), Some(password_env)) => Some((username.clone(), environment(password_env)?)), + (None, None) => None, + _ => { + return Err(TestkitError::Configuration( + "admin_username and admin_password_env must be specified together".to_owned(), + )) + } + }; + let observer = TuliproxObserver::new(api_base_url, credentials); + let policy_hash = match observer.config().await { + Ok(config) => { + if let Some(contract) = &scenario.policy_contract { + match tuliprox_testkit::policy::validate_fixture_policy(contract, &config) { + Ok(()) => {} + Err(TestkitError::Configuration(error)) => { + eprintln!("fixture policy contract rejected: {error}"); + write_report( + &scenario, + report_directory, + RunExit::Invalid, + Vec::new(), + "policy-mismatch", + run_id, + Some("policy_mismatch"), + Some(&error), + Some(2), + )?; + return Ok(RunExit::Invalid); + } + Err(error) => { + eprintln!("fixture policy evidence unavailable: {error}"); + write_report( + &scenario, + report_directory, + RunExit::Inconclusive, + Vec::new(), + "policy-unreadable", + run_id, + Some("policy_unreadable"), + Some(&error.to_string()), + Some(3), + )?; + return Ok(RunExit::Inconclusive); + } + } + } + blake3::hash(config.to_string().as_bytes()).to_hex().to_string() + } + Err(error) => { + eprintln!("configuration observation unavailable: {error}"); + write_report( + &scenario, + report_directory, + RunExit::Inconclusive, + Vec::new(), + "unavailable", + run_id, + Some("configuration_unavailable"), + Some(&error.to_string()), + Some(3), + )?; + return Ok(RunExit::Inconclusive); + } + }; + let expanded_steps = scenario.expanded_steps()?; + let requires_discovery = expanded_steps.iter().any(|step| { + step.start.as_ref().and_then(|start| start.channel.as_ref().or(start.vod_object.as_ref())).is_some() + }); + let virtual_ids_by_user = if requires_discovery { + let required_markers = channel_markers(&scenario, &expanded_steps)?; + let required_vod = vod_objects(&expanded_steps); + let mut map = HashMap::new(); + let usernames: std::collections::BTreeSet = + scenario.actors.iter().filter_map(|actor| actor.username.clone()).collect(); + if usernames.is_empty() { + let playlist_url = scenario.tuliprox.playlist_url.as_deref().ok_or_else(|| { + TestkitError::Configuration( + "channel-based or vod-based starts require tuliprox.playlist_url".to_owned(), + ) + })?; + let vids = discover_virtual_ids(playlist_url, &required_markers, &required_vod).await?; + map.insert(String::new(), vids); + } else { + for user in usernames { + let playlist_url = format!( + "{}/m3u?username={}&password={}", + scenario.tuliprox.base_url.trim_end_matches('/'), + url::form_urlencoded::byte_serialize(user.as_bytes()).collect::(), + fixture_password(), + ); + let vids = discover_virtual_ids(&playlist_url, &required_markers, &required_vod).await?; + map.insert(user, vids); + } + } + map + } else { + HashMap::new() + }; + let origin_observer = origin_control_url.map(OriginObserver::new); + let controller_state = start_controller_transport(agent_listen, &scenario.name).await?; + wait_for_required_agents(&controller_state, &scenario.agents.required).await?; + let (stop_lease_renewals, lease_stop) = watch::channel(false); + let lease_renewal_task = + start_agent_lease_renewals(controller_state.clone(), &scenario.agents.required, lease_stop); + let execution_result = execute_scenario_steps( + &scenario, + &expanded_steps, + &observer, + origin_observer.as_ref(), + &controller_state, + &virtual_ids_by_user, + ) + .await; + let _ = stop_lease_renewals.send(true); + let lease_result = lease_renewal_task + .await + .map_err(|error| TestkitError::Protocol(format!("agent lease renewal task failed: {error}")))?; + let (any_failed, observations_incomplete, events) = execution_result?; + lease_result?; + let (outcome, exit_code, error_kind, error_message) = if any_failed { + (RunExit::Failed, Some(1), Some("assertion_failed"), Some("one or more scenario steps failed")) + } else if observations_incomplete { + (RunExit::Inconclusive, Some(3), Some("incomplete_observations"), Some("scenario observations were incomplete")) + } else { + (RunExit::Passed, Some(0), None, None) + }; + write_report( + &scenario, + report_directory, + outcome, + events, + &policy_hash, + run_id, + error_kind, + error_message, + exit_code, + )?; + Ok(outcome) +} + +fn channel_markers(scenario: &Scenario, steps: &[tuliprox_testkit::config::Step]) -> Result, TestkitError> { + let mut markers = + steps + .iter() + .filter_map(|step| step.start.as_ref()) + .filter_map(|start| start.channel.as_deref()) + .map(|channel| { + scenario.channels.get(channel).map(|definition| definition.origin_marker).ok_or_else(|| { + TestkitError::Configuration(format!("channel {channel} is not defined by the scenario")) + }) + }) + .collect::, _>>()?; + markers.sort_unstable(); + markers.dedup(); + Ok(markers) +} + +fn vod_objects(steps: &[tuliprox_testkit::config::Step]) -> Vec { + let mut vods = steps + .iter() + .filter_map(|step| step.start.as_ref()) + .filter_map(|start| start.vod_object.as_deref()) + .map(str::to_owned) + .collect::>(); + vods.sort_unstable(); + vods.dedup(); + vods +} + +async fn discover_virtual_ids( + playlist_url: &str, + required_markers: &[u32], + required_vod_objects: &[String], +) -> Result { + let deadline = tokio::time::Instant::now() + PLAYLIST_DISCOVERY_TIMEOUT; + let client = reqwest::Client::new(); + loop { + let last_reason = match client.get(playlist_url).send().await { + Ok(response) => match response.error_for_status() { + Ok(response) => match response.text().await { + Ok(playlist) => match VirtualIdMap::from_m3u(&playlist) { + Ok(virtual_ids) => { + let missing = required_markers + .iter() + .copied() + .filter(|marker| virtual_ids.playback_url(*marker).is_err()) + .collect::>(); + let missing_vod = required_vod_objects + .iter() + .filter(|name| virtual_ids.named_playback_url(name).is_err()) + .collect::>(); + if missing.is_empty() && missing_vod.is_empty() { + return Ok(virtual_ids); + } + format!("playlist has not published markers {missing:?} or vod {missing_vod:?}") + } + Err(error) => error.to_string(), + }, + Err(error) => error.to_string(), + }, + Err(error) => error.to_string(), + }, + Err(error) => error.to_string(), + }; + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!( + "Tuliprox did not publish the required virtual IDs within {} seconds: {last_reason}", + PLAYLIST_DISCOVERY_TIMEOUT.as_secs() + ))); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +fn start_agent_lease_renewals( + controller_state: ControllerState, + agent_ids: &[String], + mut stop: watch::Receiver, +) -> tokio::task::JoinHandle> { + const LEASE_DURATION: Duration = Duration::from_secs(15); + let agents = agent_ids.iter().cloned().map(AgentId::new).collect::>(); + tokio::spawn(async move { + for agent in &agents { + dispatch_run_lease(&controller_state, agent, LEASE_DURATION, true).await?; + } + let mut interval = tokio::time::interval(LEASE_DURATION / 3); + interval.tick().await; + loop { + tokio::select! { + changed = stop.changed() => { + if changed.is_err() || *stop.borrow() { + return Ok(()); + } + } + _ = interval.tick() => { + for agent in &agents { + dispatch_run_lease(&controller_state, agent, LEASE_DURATION, false).await?; + } + } + } + } + }) +} + +async fn dispatch_run_lease( + controller_state: &ControllerState, + agent: &AgentId, + lease_duration: Duration, + initial: bool, +) -> Result<(), TestkitError> { + let sequence = controller_state.next_command_sequence(); + let command_id = format!("lease-{}-{sequence}", agent.0); + let lease_millis = u64::try_from(lease_duration.as_millis()).unwrap_or(u64::MAX); + controller_state + .dispatch( + agent, + Envelope { + schema_version: 1, + run_id: controller_state.run_id(), + run_generation: 1, + message_id: command_id.clone(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: sequence, + caused_by_command_id: Some(CommandId::new(command_id)), + local_elapsed_nanos: 0, + payload: if initial { + ControlCommand::ConfigureRun { lease_millis } + } else { + ControlCommand::RenewRunLease { lease_millis } + }, + }, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn check_origin_assertions<'a>( + step_id: &'a str, + assert_origin: &tuliprox_testkit::config::AssertOrigin, + obs: &OriginObserver, + origin_run_id: &str, + events: &mut Vec<(&'a str, &'static str)>, + any_failed: &mut bool, + observations_incomplete: &mut bool, +) { + let deadline = tokio::time::Instant::now() + Duration::from_millis(2000); + loop { + if let Ok(stats) = obs.stats(origin_run_id).await { + let body_matches = + assert_origin.active_body.is_none_or(|expected| stats.active_body_connections == expected); + if body_matches || tokio::time::Instant::now() >= deadline { + if let Some(max_tcp) = assert_origin.max_active_tcp { + if stats.max_active_tcp_connections > max_tcp { + *any_failed = true; + events.push((step_id, "origin_peak_tcp_exceeded")); + } + } + if let Some(max_body) = assert_origin.max_active_body { + if stats.max_active_body_connections > max_body { + *any_failed = true; + events.push((step_id, "origin_peak_body_exceeded")); + } + } + if let Some(act_body) = assert_origin.active_body { + if stats.active_body_connections != act_body { + *any_failed = true; + events.push((step_id, "origin_active_body_mismatch")); + } + } + let check_evictions = assert_origin.no_evictions.unwrap_or(false); + let check_rejections = assert_origin.no_limit_rejections.unwrap_or(false); + if check_evictions || check_rejections { + if let Ok(evts) = obs.events(origin_run_id).await { + if check_evictions + && evts.iter().any(|e| matches!(e.kind, OriginEventKind::EvictionTriggered { .. })) + { + *any_failed = true; + events.push((step_id, "origin_eviction_observed")); + } + if check_rejections + && evts.iter().any(|e| matches!(e.kind, OriginEventKind::LimitRejected { .. })) + { + *any_failed = true; + events.push((step_id, "origin_rejection_observed")); + } + } else { + *observations_incomplete = true; + events.push((step_id, "origin_events_unavailable")); + } + } + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } else { + if tokio::time::Instant::now() >= deadline { + *observations_incomplete = true; + events.push((step_id, "origin_stats_unavailable")); + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + } +} + +#[allow(clippy::too_many_lines)] +async fn execute_scenario_steps<'a>( + scenario: &'a Scenario, + steps: &'a [tuliprox_testkit::config::Step], + observer: &TuliproxObserver, + origin_observer: Option<&OriginObserver>, + controller_state: &ControllerState, + virtual_ids_by_user: &HashMap, +) -> Result<(bool, bool, Vec<(&'a str, &'static str)>), TestkitError> { + let mut any_failed = false; + let mut observations_incomplete = false; + let mut events = Vec::new(); + let mut held_playbacks: HashMap = HashMap::new(); + let mut remote_playbacks: HashMap = HashMap::new(); + let mut expected_terminations = HashSet::new(); + let mut playback_users = HashMap::new(); + let mut admission_oracles = scenario.policy_contract.as_ref().map(|contract| { + contract + .users + .iter() + .map(|(username, policy)| { + ( + username.clone(), + AdmissionOracle::with_policy( + policy.max_connections, + policy.soft_connections, + contract.effective_admission_strategies(), + contract.grace.as_ref().map(|grace| grace.mode), + ), + ) + }) + .collect::>() + }); + let mut admission_order = 0_u64; + if let Some(obs) = origin_observer { + let origin_run_id = + scenario.origin.as_ref().map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + let _ = obs.reset(&origin_run_id.0).await; + } + let execution_result = async { + for step in steps { + if let Some(stop) = &step.stop { + if let Some(oracles) = admission_oracles.as_mut() { + if let Some(username) = playback_users.remove(&stop.playback_id) { + oracles + .get_mut(&username) + .ok_or_else(|| TestkitError::Protocol(format!("missing admission oracle for {username}")))? + .release(&stop.playback_id)?; + } + } + if let Some(held) = held_playbacks.remove(&stop.playback_id) { + let _ = held.release.send(()); + match held.task.await { + Ok(Ok(PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. })) => { + events.push((stop.playback_id.as_str(), "stopped")); + } + Ok(Ok(_) | Err(_)) | Err(_) => any_failed = true, + } + } else if let Some(agent) = remote_playbacks.remove(&stop.playback_id) { + let outcome = dispatch_remote_stop( + controller_state, + &agent, + &scenario.name, + &step.command_id, + &stop.playback_id, + ) + .await?; + match outcome { + PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. } => { + events.push((stop.playback_id.as_str(), "stopped")); + } + _ => any_failed = true, + } + } else { + return Err(TestkitError::Protocol(format!( + "playback {} was not active at stop", + stop.playback_id + ))); + } + + if let Some(assert_origin) = &step.assert_origin { + if let Some(obs) = origin_observer { + let origin_run_id = scenario + .origin + .as_ref() + .map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + check_origin_assertions( + stop.playback_id.as_str(), + assert_origin, + obs, + &origin_run_id.0, + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + } else { + observations_incomplete = true; + events.push((stop.playback_id.as_str(), "origin_observer_missing")); + } + } + + continue; + } + let start = + step.start.as_ref().ok_or_else(|| TestkitError::Configuration("step has no start".to_owned()))?; + let actor = scenario + .actors + .iter() + .find(|actor| actor.id == start.actor) + .ok_or_else(|| TestkitError::Configuration(format!("unknown actor {}", start.actor)))?; + let is_vod = start.vod_object.is_some() + || start.range.is_some() + || start.read_limit_bytes.is_some() + || start.method.is_some(); + let frames = step + .await_frames + .or_else(|| step.await_condition.as_ref().and_then(|condition| condition.valid_frames)) + .unwrap_or(1); + let url = if let Some(url) = start.url.as_deref() { + url + } else { + let vids = if let Some(username) = actor.username.as_deref() { + virtual_ids_by_user.get(username).or_else(|| virtual_ids_by_user.get("")) + } else { + virtual_ids_by_user.get("") + } + .ok_or_else(|| { + TestkitError::Configuration(format!( + "virtual-ID discovery was not initialized for actor {}", + actor.id + )) + })?; + if let Some(vod) = start.vod_object.as_deref() { + vids.named_playback_url(vod)? + } else { + let channel = start.channel.as_deref().ok_or_else(|| { + TestkitError::Configuration("start has no URL, vod_object, or channel".to_owned()) + })?; + let marker = scenario + .channels + .get(channel) + .ok_or_else(|| TestkitError::Configuration(format!("unknown channel {channel}")))? + .origin_marker; + vids.playback_url(marker)? + } + }; + let marker = if is_vod { + 0 + } else { + start + .channel + .as_deref() + .and_then(|channel| scenario.channels.get(channel)) + .map_or_else(|| marker_from_url(url), |channel| Ok(channel.origin_marker))? + }; + admission_order = admission_order.saturating_add(1); + let mut evicted_victim = None; + let mut evicted_victim_id = None; + if let Some(oracles) = admission_oracles.as_mut() { + let username = actor.username.as_deref().ok_or_else(|| { + TestkitError::Configuration(format!( + "actor {} requires username because this scenario has a user admission policy", + actor.id + )) + })?; + let oracle = oracles.get_mut(username).ok_or_else(|| { + TestkitError::Configuration(format!( + "actor {} references user {username} absent from policy_contract", + actor.id + )) + })?; + let client_ip = + actor.client_ip.as_ref().and_then(|ip| ip.value.as_deref()).unwrap_or("peer-address").to_owned(); + let (decision, evicted) = oracle.admit_with_evicted(AdmissionRequest { + playback_id: start.playback_id.clone(), + client_ip, + started_order: admission_order, + })?; + evicted_victim = evicted; + let oracle_expected = match &decision { + AdmissionDecision::Admit + | AdmissionDecision::AdmitSoft + | AdmissionDecision::Evict { .. } + | AdmissionDecision::Grace { .. } => ExpectedPlayback::Streaming, + AdmissionDecision::Deny => ExpectedPlayback::Rejected, + }; + let oracle_tracks_playback = matches!( + &decision, + AdmissionDecision::Admit | AdmissionDecision::AdmitSoft | AdmissionDecision::Evict { .. } + ); + if let AdmissionDecision::Evict { playback_id } = &decision { + expected_terminations.insert(playback_id.clone()); + evicted_victim_id = Some(playback_id.clone()); + } + let provider_limited_rejection = + scenario.policy_contract.as_ref().and_then(|contract| contract.provider_max_connections).is_some() + && step.expect.is_rejected() + && oracle_expected == ExpectedPlayback::Streaming; + if step.expect != oracle_expected && !provider_limited_rejection { + return Err(TestkitError::Configuration(format!( + "step {} expects {:?}, but policy oracle expects {:?}", + step.command_id, step.expect, oracle_expected + ))); + } + if oracle_tracks_playback { + playback_users.insert(start.playback_id.clone(), username.to_owned()); + } + } + let mut headers = actor_request_headers(actor, actor.agent == "local")?; + if let Some(ua) = &start.user_agent { + headers.insert("user-agent".to_owned(), ua.clone()); + } + let origin_run_id = scenario + .origin + .as_ref() + .map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + let local_outcome = if actor.agent == "local" { + if is_vod { + match start_held_vod_playback( + start.playback_id.clone(), + url, + start.vod_object.clone(), + start.method.clone(), + start.range.clone(), + start.expected_status, + start.expected_content_range.clone(), + start.read_limit_bytes, + start.post_read_action.unwrap_or_default(), + &headers, + ) + .await? + { + Ok(held) => { + held_playbacks.insert(start.playback_id.clone(), held); + Some(PlaybackOutcome::Streaming { frames: 0, bytes: 0 }) + } + Err(outcome) => Some(outcome), + } + } else { + match start_held_playback(start.playback_id.clone(), &origin_run_id, url, marker, frames, &headers) + .await? + { + Ok(held) => { + held_playbacks.insert(start.playback_id.clone(), held); + Some(PlaybackOutcome::Streaming { frames, bytes: 0 }) + } + Err(outcome) => Some(outcome), + } + } + } else { + None + }; + let remote_outcome = if actor.agent == "local" { + None + } else { + dispatch_remote_playback( + controller_state, + &actor.agent, + &scenario.name, + &step.command_id, + &start.playback_id, + url, + headers.clone(), + Some(origin_run_id), + Some(marker), + Some(frames), + ) + .await? + }; + let (step_passed, actual_streaming) = match (&local_outcome, &remote_outcome) { + (Some(outcome), _) | (None, Some(outcome)) => { + (step.expect.matches_outcome(outcome), outcome.is_streaming()) + } + (None, None) => (false, false), + }; + if step_passed { + events.push(( + start.playback_id.as_str(), + if step.expect.is_streaming() { "received_valid_frames" } else { "rejected_as_expected" }, + )); + if remote_outcome.as_ref().is_some_and(PlaybackOutcome::is_streaming) { + remote_playbacks.insert(start.playback_id.clone(), actor.agent.clone()); + } + } else { + eprintln!( + "Step {} ({}) failed: expect={:?}, local_outcome={:?}, remote_outcome={:?}", + step.command_id, start.playback_id, step.expect, local_outcome, remote_outcome + ); + any_failed = true; + events.push((start.playback_id.as_str(), "failed")); + } + if !actual_streaming { + if let Some(oracles) = admission_oracles.as_mut() { + if let Some(username) = actor.username.as_deref() { + if let Some(oracle) = oracles.get_mut(username) { + oracle.rollback(&start.playback_id, evicted_victim); + } + } + playback_users.remove(&start.playback_id); + if let Some(victim_id) = &evicted_victim_id { + expected_terminations.remove(victim_id); + } + } + } + if let Some(assert_origin) = &step.assert_origin { + if let Some(obs) = origin_observer { + let origin_run_id = scenario + .origin + .as_ref() + .map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + check_origin_assertions( + start.playback_id.as_str(), + assert_origin, + obs, + &origin_run_id.0, + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + } else { + observations_incomplete = true; + events.push((start.playback_id.as_str(), "origin_observer_missing")); + } + } + if let Ok(snapshot) = observer.runtime_snapshot().await { + events.push((start.playback_id.as_str(), "runtime_observed")); + if let Some(policy) = &scenario.policy_contract { + if let Ok(slots) = provider_slot_count(&snapshot.status) { + if policy.provider_max_connections.is_some_and(|limit| slots > usize::from(limit)) { + any_failed = true; + events.push((start.playback_id.as_str(), "provider_limit_exceeded")); + } + if policy.expected_provider_slots.is_some_and(|expected| slots != expected) { + any_failed = true; + events.push((start.playback_id.as_str(), "provider_slot_count_mismatch")); + } + } else { + observations_incomplete = true; + events.push((start.playback_id.as_str(), "provider_observation_invalid")); + } + } + } else { + observations_incomplete = true; + events.push((start.playback_id.as_str(), "runtime_observation_unavailable")); + } + } + Ok::<(), TestkitError>(()) + } + .await; + for (_, mut held) in held_playbacks { + let expected_termination = expected_terminations.contains(&held.playback_id); + let event_playback_id = steps + .iter() + .filter_map(|step| step.start.as_ref()) + .find(|start| start.playback_id == held.playback_id) + .map(|start| start.playback_id.as_str()) + .ok_or_else(|| TestkitError::Protocol(format!("missing scenario playback {}", held.playback_id)))?; + if expected_termination { + match tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, &mut held.task).await { + Ok(Ok(Ok(PlaybackOutcome::Streaming { .. } | PlaybackOutcome::ExplicitStop))) => { + events.push((event_playback_id, "unexpected_clean_terminal")); + any_failed = true; + } + Ok(Ok(Ok(_))) => events.push((event_playback_id, "evicted")), + Ok(Ok(Err(_)) | Err(_)) => { + events.push((event_playback_id, "unexpected_eviction_terminal")); + any_failed = true; + } + Err(_) => { + held.task.abort(); + let _ = held.task.await; + any_failed = true; + events.push((event_playback_id, "expected_eviction_not_observed")); + } + } + let _ = held.release.is_closed(); + } else { + let _ = held.release.send(()); + match held.task.await { + Ok(Ok(PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. })) => {} + Ok(Ok(_) | Err(_)) | Err(_) => any_failed = true, + } + } + } + for (playback_id, agent) in remote_playbacks { + if let Ok(outcome) = dispatch_remote_stop( + controller_state, + &agent, + &scenario.name, + &format!("cleanup-{playback_id}"), + &playback_id, + ) + .await + { + match outcome { + PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. } => {} + _ => any_failed = true, + } + } else { + any_failed = true; + } + } + execution_result?; + if scenario.policy_contract.as_ref().is_some_and(|contract| contract.provider_max_connections.is_some()) { + if wait_for_provider_slots(observer, 0, Duration::from_secs(5)).await.is_ok() { + events.push((scenario.name.as_str(), "provider_slots_released")); + } else { + any_failed = true; + events.push((scenario.name.as_str(), "provider_slots_not_released")); + } + } + Ok((any_failed, observations_incomplete, events)) +} + +async fn wait_for_provider_slots( + observer: &TuliproxObserver, + expected: usize, + timeout: Duration, +) -> Result<(), TestkitError> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if provider_slot_count(&observer.status().await?)? == expected { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!("provider slots did not converge to {expected}"))); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +fn provider_slot_count(status: &serde_json::Value) -> Result { + let providers = status.get("active_provider_connections").ok_or_else(|| { + TestkitError::Protocol("status response missing active_provider_connections field".to_owned()) + })?; + if providers.is_null() { + return Ok(0); + } + let Some(providers) = providers.as_object() else { + return Err(TestkitError::Protocol("status active_provider_connections is not an object".to_owned())); + }; + providers.values().try_fold(0_usize, |total, value| { + let count = value + .as_u64() + .ok_or_else(|| TestkitError::Protocol("provider connection count is not an unsigned integer".to_owned()))?; + let count = usize::try_from(count) + .map_err(|_| TestkitError::Protocol("provider connection count exceeds platform capacity".to_owned()))?; + total.checked_add(count).ok_or_else(|| TestkitError::Protocol("provider connection count overflow".to_owned())) + }) +} + +fn actor_request_headers( + actor: &tuliprox_testkit::config::Actor, + allow_controller_secrets: bool, +) -> Result, TestkitError> { + let mut headers = BTreeMap::new(); + if let Some(user_agent) = &actor.user_agent { + headers.insert("user-agent".to_owned(), user_agent.clone()); + } + if let Some(client_ip) = &actor.client_ip { + let value = client_ip.value.as_deref().filter(|value| !value.trim().is_empty()).ok_or_else(|| { + TestkitError::Configuration(format!("actor {} client_ip requires a non-empty value", actor.id)) + })?; + match client_ip.mode.as_str() { + "x-real-ip" => { + headers.insert("x-real-ip".to_owned(), value.to_owned()); + } + "forwarded" | "x-forwarded-for" => { + headers.insert("x-forwarded-for".to_owned(), value.to_owned()); + } + unsupported => { + return Err(TestkitError::Configuration(format!( + "actor {} has unsupported client_ip mode {unsupported}", + actor.id + ))); + } + } + } + match (&actor.username, &actor.password_env) { + (Some(username), Some(password_env)) if allow_controller_secrets => { + let credentials = format!("{username}:{}", environment(password_env)?); + headers.insert("authorization".to_owned(), format!("Basic {}", BASE64.encode(credentials))); + } + (Some(_), Some(_)) => { + return Err(TestkitError::Configuration(format!( + "remote actor {} credentials must be supplied by an agent-local secret provider", + actor.id + ))); + } + (Some(_) | None, None) => {} + _ => { + return Err(TestkitError::Configuration(format!( + "actor {} must specify username and password_env together", + actor.id + ))); + } + } + Ok(headers) +} + +async fn start_controller_transport(agent_listen: SocketAddr, run_name: &str) -> Result { + let listener = TcpListener::bind(agent_listen).await?; + let state: ControllerState = Arc::new(SharedControllerState::new(RunId::new(run_name), 1)); + let app = Router::new().route("/agent", get(controller_websocket)).with_state(state.clone()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok(state) +} + +async fn wait_for_required_agents(controller_state: &ControllerState, required: &[String]) -> Result<(), TestkitError> { + if required.is_empty() { + return Ok(()); + } + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let connected = controller_state + .ready_agents() + .await + .into_iter() + .map(|agent| agent.0) + .collect::>(); + if required.iter().all(|agent| connected.contains(agent)) { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol("required test agents did not become ready before deadline".to_owned())); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[allow(clippy::too_many_arguments)] +async fn dispatch_remote_playback( + controller_state: &ControllerState, + agent: &str, + run_name: &str, + command_id: &str, + playback_id: &str, + url: &str, + headers: BTreeMap, + expected_run_id: Option, + expected_marker: Option, + required_frames: Option, +) -> Result, TestkitError> { + let command = Envelope { + schema_version: 1, + run_id: RunId::new(run_name), + run_generation: 1, + message_id: command_id.to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: controller_state.next_command_sequence(), + caused_by_command_id: Some(tuliprox_testkit::protocol::CommandId::new(command_id)), + local_elapsed_nanos: 0, + payload: ControlCommand::StartPlayback { + playback_id: tuliprox_testkit::protocol::PlaybackId::new(playback_id), + url: url.to_owned(), + headers, + expected_run_id, + expected_marker, + required_frames, + }, + }; + controller_state.dispatch(&AgentId::new(agent), command).await?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some(event) = controller_state.take_playback_event(playback_id).await { + match event { + PlaybackEvent::FirstValidFrame { .. } => { + return Ok(Some(PlaybackOutcome::Streaming { frames: required_frames.unwrap_or(5), bytes: 0 })); + } + PlaybackEvent::Terminal { typed_outcome, outcome, .. } => { + return Ok(Some(typed_outcome.unwrap_or_else(|| match outcome.as_str() { + "passed" => PlaybackOutcome::Streaming { frames: required_frames.unwrap_or(5), bytes: 0 }, + "rejected" | "rejected_as_expected" => { + PlaybackOutcome::AdmissionRejected { reason: RejectionReason::Other("rejected".to_owned()) } + } + _ => PlaybackOutcome::InfrastructureError { message: outcome }, + }))); + } + PlaybackEvent::CommandAccepted { .. } + | PlaybackEvent::HeadersReceived { .. } + | PlaybackEvent::Progress { .. } + | PlaybackEvent::LeaseExpired => {} + } + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!( + "remote agent {agent} did not report terminal playback {playback_id}" + ))); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn dispatch_remote_stop( + controller_state: &ControllerState, + agent: &str, + run_name: &str, + command_id: &str, + playback_id: &str, +) -> Result { + controller_state + .dispatch( + &AgentId::new(agent), + Envelope { + schema_version: 1, + run_id: RunId::new(run_name), + run_generation: 1, + message_id: command_id.to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: controller_state.next_command_sequence(), + caused_by_command_id: Some(tuliprox_testkit::protocol::CommandId::new(command_id)), + local_elapsed_nanos: 0, + payload: ControlCommand::StopPlayback { + playback_id: tuliprox_testkit::protocol::PlaybackId::new(playback_id), + }, + }, + ) + .await?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some(PlaybackEvent::Terminal { typed_outcome, outcome, .. }) = + controller_state.take_playback_event(playback_id).await + { + return Ok(typed_outcome.unwrap_or(match outcome.as_str() { + "passed" | "cancelled" => PlaybackOutcome::ExplicitStop, + _ => PlaybackOutcome::InfrastructureError { message: outcome }, + })); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!( + "remote agent {agent} did not confirm stop for playback {playback_id}" + ))); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[allow(clippy::too_many_arguments)] +fn write_raw_report( + scenario_name: &str, + directory: &std::path::Path, + outcome: RunExit, + events: Vec<(&str, &str)>, + policy_hash: &str, + run_id: Option<&str>, + error_kind: Option<&str>, + error_message: Option<&str>, + exit_code: Option, +) -> Result<(), TestkitError> { + let outcome_text = match outcome { + RunExit::Passed => "passed", + RunExit::Failed => "failed", + RunExit::Invalid => "invalid", + RunExit::Inconclusive => "inconclusive", + }; + let report = RunReport { + schema_version: 1, + scenario: scenario_name, + outcome: outcome_text, + policy_hash, + run_id, + error_kind, + error_message, + exit_code, + events: events + .into_iter() + .map(|(playback_id, event)| ReportEvent { source: "agent", playback_id, event, detail: "redacted" }) + .collect(), + }; + report.write_to(directory) +} + +#[allow(clippy::too_many_arguments)] +fn write_report( + scenario: &Scenario, + directory: &std::path::Path, + outcome: RunExit, + events: Vec<(&str, &str)>, + policy_hash: &str, + run_id: Option<&str>, + error_kind: Option<&str>, + error_message: Option<&str>, + exit_code: Option, +) -> Result<(), TestkitError> { + write_raw_report( + &scenario.name, + directory, + outcome, + events, + policy_hash, + run_id, + error_kind, + error_message, + exit_code, + ) +} + +fn marker_from_url(url: &str) -> Result { + let parsed = + url::Url::parse(url).map_err(|error| TestkitError::Configuration(format!("invalid stream URL: {error}")))?; + let path = parsed + .path_segments() + .ok_or_else(|| TestkitError::Configuration("stream URL has no path segments".to_owned()))? + .collect::>(); + path.iter() + .rev() + .find_map(|segment| segment.split('.').next().and_then(|value| value.parse::().ok())) + .ok_or_else(|| TestkitError::Configuration("stream URL has no numeric marker path segment".to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_origin_state(run_id: &str) -> OriginState { + OriginState { + run_id: RunId::new(run_id), + bitrate: 64_000, + markers: Arc::new(vec![17]), + stream_counter: Arc::new(Mutex::new(4)), + observations: Arc::new(Mutex::new(Vec::new())), + faults: Arc::new(Mutex::new(FaultSchedule::default())), + hls_sequences: Arc::new(Mutex::new(HashMap::from([(17, 9)]))), + tracker: OriginTracker::new(run_id, OriginPolicy::default()), + } + } + + #[test] + fn extracts_marker_from_ts_url() { + assert!(matches!(marker_from_url("http://origin/live/17.ts"), Ok(17))); + } + + #[test] + fn extracts_marker_from_hls_manifest_url() { + assert!(matches!(marker_from_url("http://origin/hls/17/index.m3u8"), Ok(17))); + } + + #[test] + fn recognizes_hls_manifest_urls() { + assert!(is_hls_manifest_url("http://origin/hls/17/index.m3u8")); + assert!(!is_hls_manifest_url("http://origin/live/17.ts")); + } + + #[test] + fn parses_open_and_closed_byte_ranges() { + assert_eq!(tuliprox_testkit::vod::parse_range_header("bytes=2-4", 10), Some((2, 5))); + assert_eq!(tuliprox_testkit::vod::parse_range_header("bytes=8-", 10), Some((8, 10))); + assert_eq!(tuliprox_testkit::vod::parse_range_header("bytes=10-", 10), None); + } + + #[test] + fn vod_validators_are_stable_for_identical_payloads() { + let payload = vod_payload("movie.ts"); + let mut first = StatusCode::OK.into_response(); + let mut second = StatusCode::OK.into_response(); + apply_vod_validators(&mut first, &payload); + apply_vod_validators(&mut second, &payload); + assert_eq!(first.headers().get(header::ETAG), second.headers().get(header::ETAG)); + assert_eq!( + first.headers().get(header::LAST_MODIFIED), + Some(&HeaderValue::from_static("Thu, 01 Jan 1970 00:00:00 GMT")) + ); + } + + #[tokio::test] + async fn origin_reset_is_scoped_to_its_run() { + let state = test_origin_state("run-a"); + state.observations.lock().await.push(OriginObservation { + origin_stream_id: "origin-4".to_owned(), + channel_marker: 17, + frames_emitted: 0, + bytes_emitted: 0, + }); + assert_eq!(reset_origin_run(Path("other".to_owned()), State(state.clone())).await, StatusCode::NOT_FOUND); + assert_eq!(reset_origin_run(Path("run-a".to_owned()), State(state.clone())).await, StatusCode::NO_CONTENT); + assert!(state.observations.lock().await.is_empty()); + assert_eq!(*state.stream_counter.lock().await, 0); + assert!(state.hls_sequences.lock().await.is_empty()); + } + + #[tokio::test] + async fn catalog_exposes_each_configured_marker() { + let mut state = test_origin_state("catalog-run"); + state.markers = Arc::new(vec![17, 19]); + let document = catalog_m3u(&state, "127.0.0.1"); + assert!(document.contains("tvg-id=\"test-17\"")); + assert!(document.contains("tvg-id=\"test-19\"")); + assert!(document.contains("run=catalog-run")); + } + + #[tokio::test] + async fn hls_rejects_a_marker_absent_from_the_origin_catalog() { + let state = test_origin_state("hls-run"); + assert_eq!( + hls( + Path("19/index.m3u8".to_owned()), + Query(HashMap::from([("run".to_owned(), "hls-run".to_owned())])), + State(state), + ) + .await + .status(), + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn live_rejects_a_missing_or_foreign_run_correlation() { + let state = test_origin_state("live-run"); + assert_eq!( + live(Path("17.ts".to_owned()), Query(HashMap::new()), State(state.clone()), None, HeaderMap::new()) + .await + .status(), + StatusCode::NOT_FOUND + ); + assert_eq!( + live( + Path("17.ts".to_owned()), + Query(HashMap::from([("run".to_owned(), "other".to_owned())])), + State(state), + None, + HeaderMap::new(), + ) + .await + .status(), + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn vod_head_rejects_a_missing_run_correlation() { + let state = test_origin_state("vod-run"); + assert_eq!( + vod_head(Path("movie.ts".to_owned()), Query(HashMap::new()), State(state)).await.status(), + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn faults_cannot_be_mutated_by_a_foreign_run() { + let state = test_origin_state("fault-run"); + assert_eq!( + set_fault(Path(("other".to_owned(), "close-before-first-byte".to_owned())), State(state.clone()),).await, + StatusCode::NOT_FOUND + ); + assert!(state.faults.lock().await.take("close-before-first-byte").is_none()); + } + + #[test] + fn actor_identity_uses_configured_forwarded_ip_and_user_agent() { + let actor = tuliprox_testkit::config::Actor { + id: "viewer".to_owned(), + agent: "local".to_owned(), + username: None, + password_env: None, + user_agent: Some("test-receiver/1".to_owned()), + client_ip: Some(tuliprox_testkit::config::ClientIp { + mode: "x-real-ip".to_owned(), + value: Some("198.51.100.42".to_owned()), + }), + }; + let headers = actor_request_headers(&actor, true).unwrap(); + assert_eq!(headers.get("x-real-ip"), Some(&"198.51.100.42".to_owned())); + assert_eq!(headers.get("user-agent"), Some(&"test-receiver/1".to_owned())); + } + + #[test] + fn remote_actor_cannot_receive_controller_credentials() { + let actor = tuliprox_testkit::config::Actor { + id: "viewer".to_owned(), + agent: "agent-b".to_owned(), + username: Some("alice".to_owned()), + password_env: Some("TESTKIT_PASSWORD".to_owned()), + user_agent: None, + client_ip: None, + }; + assert!(matches!(actor_request_headers(&actor, false), Err(TestkitError::Configuration(_)))); + } + + #[tokio::test] + async fn vod_reader_rejects_206_missing_content_range() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buf = [0u8; 1024]; + let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut buf).await; + let response = "HTTP/1.1 206 Partial Content\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntest"; + let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await; + }); + + let (ready_tx, _ready_rx) = tokio::sync::oneshot::channel(); + let (_release_tx, release_rx) = tokio::sync::oneshot::channel(); + let outcome = run_vod_until_released( + format!("http://127.0.0.1:{port}/movie.mkv"), + None, + None, + Some("bytes=0-3".to_owned()), + None, + None, + None, + PostReadAction::KeepOpen, + BTreeMap::new(), + ready_tx, + release_rx, + ) + .await + .unwrap(); + + assert!(matches!(outcome, PlaybackOutcome::InvalidData { message } if message.contains("omits Content-Range"))); + } + + #[tokio::test] + async fn check_origin_assertions_events_http_500_marks_observations_incomplete() { + use axum::routing::get; + let app = Router::new() + .route( + "/v1/runs/test-run/stats", + get(|| async { + Json(OriginStats { + active_tcp_connections: 0, + max_active_tcp_connections: 0, + active_body_connections: 0, + max_active_body_connections: 0, + total_tcp_connections: 0, + total_requests: 0, + total_bytes_emitted: 0, + }) + }), + ) + .route("/v1/runs/test-run/events", get(|| async { StatusCode::INTERNAL_SERVER_ERROR })); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let obs = OriginObserver::new(&format!("http://127.0.0.1:{port}")); + let mut events = Vec::new(); + let mut any_failed = false; + let mut observations_incomplete = false; + let assert_origin = tuliprox_testkit::config::AssertOrigin { + active_body: None, + max_active_body: None, + max_active_tcp: None, + no_evictions: Some(true), + no_limit_rejections: None, + }; + + check_origin_assertions( + "step-1", + &assert_origin, + &obs, + "test-run", + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + + assert!(!any_failed); + assert!(observations_incomplete); + assert!(events.iter().any(|(s, e)| *s == "step-1" && *e == "origin_events_unavailable")); + } + + #[tokio::test] + async fn check_origin_assertions_eviction_observed_marks_failed() { + use axum::routing::get; + let app = Router::new() + .route( + "/v1/runs/test-run/stats", + get(|| async { + Json(OriginStats { + active_tcp_connections: 0, + max_active_tcp_connections: 0, + active_body_connections: 0, + max_active_body_connections: 0, + total_tcp_connections: 0, + total_requests: 0, + total_bytes_emitted: 0, + }) + }), + ) + .route( + "/v1/runs/test-run/events", + get(|| async { + Json(vec![OriginEvent::new( + 1, + "test-run", + OriginEventKind::EvictionTriggered { + evicted_conn_id: 1, + evicted_request_id: 1, + triggering_conn_id: 2, + }, + )]) + }), + ); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let obs = OriginObserver::new(&format!("http://127.0.0.1:{port}")); + let mut events = Vec::new(); + let mut any_failed = false; + let mut observations_incomplete = false; + let assert_origin = tuliprox_testkit::config::AssertOrigin { + active_body: None, + max_active_body: None, + max_active_tcp: None, + no_evictions: Some(true), + no_limit_rejections: None, + }; + + check_origin_assertions( + "step-1", + &assert_origin, + &obs, + "test-run", + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + + assert!(any_failed); + assert!(!observations_incomplete); + assert!(events.iter().any(|(s, e)| *s == "step-1" && *e == "origin_eviction_observed")); + } +} diff --git a/tools/tuliprox-testkit/src/observation.rs b/tools/tuliprox-testkit/src/observation.rs new file mode 100644 index 000000000..bba8f843b --- /dev/null +++ b/tools/tuliprox-testkit/src/observation.rs @@ -0,0 +1,49 @@ +use crate::TestkitError; +use serde_json::Value; + +#[derive(Debug, Clone)] +pub struct RuntimeSnapshot { + pub status: Value, + pub streams: Value, + pub history: Value, +} + +#[derive(Clone)] +pub struct TuliproxObserver { + base_url: String, + client: reqwest::Client, + credentials: Option<(String, String)>, +} + +impl TuliproxObserver { + #[must_use] + pub fn new(base_url: impl Into, credentials: Option<(String, String)>) -> Self { + Self { base_url: base_url.into().trim_end_matches('/').to_owned(), client: reqwest::Client::new(), credentials } + } + + pub async fn config(&self) -> Result { self.get_json("config").await } + + pub async fn status(&self) -> Result { self.get_json("status").await } + + pub async fn streams(&self) -> Result { self.get_json("streams").await } + + pub async fn stream_history(&self) -> Result { self.get_json("stream-history").await } + + pub async fn runtime_snapshot(&self) -> Result { + let (status, streams) = tokio::join!(self.status(), self.streams()); + Ok(RuntimeSnapshot { status: status?, streams: streams?, history: Value::Null }) + } + + async fn get_json(&self, endpoint: &str) -> Result { + let request = self.client.get(format!("{}/{}", self.base_url, endpoint)); + let request = match &self.credentials { + Some((username, password)) => request.basic_auth(username, Some(password)), + None => request, + }; + let response = request.send().await?; + if !response.status().is_success() { + return Err(TestkitError::Protocol(format!("observer endpoint {endpoint} returned {}", response.status()))); + } + response.json().await.map_err(TestkitError::Http) + } +} diff --git a/tools/tuliprox-testkit/src/oracle.rs b/tools/tuliprox-testkit/src/oracle.rs new file mode 100644 index 000000000..fe13078a2 --- /dev/null +++ b/tools/tuliprox-testkit/src/oracle.rs @@ -0,0 +1,237 @@ +use crate::TestkitError; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AdmissionStrategy { + EvictUserSameIpOldest, + EvictUserSameIpLatest, + EvictUserOldest, + EvictUserLatest, + GraceInstantStream, + GraceHoldStream, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GraceMode { + Instant, + HoldStream, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Request { + pub playback_id: String, + pub client_ip: String, + pub started_order: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decision { + Admit, + AdmitSoft, + Deny, + Evict { playback_id: String }, + Grace { mode: GraceMode }, +} + +#[derive(Debug)] +pub struct AdmissionOracle { + max_connections: usize, + soft_connections: usize, + strategies: Vec, + grace: Option, + active: HashMap, +} + +impl AdmissionOracle { + #[must_use] + pub fn new(max_connections: usize, strategies: Vec) -> Self { + Self { max_connections, soft_connections: 0, strategies, grace: None, active: HashMap::new() } + } + + #[must_use] + pub fn with_policy( + max_connections: usize, + soft_connections: usize, + strategies: Vec, + grace: Option, + ) -> Self { + Self { max_connections, soft_connections, strategies, grace, active: HashMap::new() } + } + + pub fn admit_with_evicted(&mut self, request: Request) -> Result<(Decision, Option), TestkitError> { + if self.active.contains_key(&request.playback_id) { + return Err(TestkitError::Configuration("duplicate oracle playback".to_owned())); + } + if self.active.len() < self.max_connections { + self.active.insert(request.playback_id.clone(), request); + return Ok((Decision::Admit, None)); + } + if self.active.len() < self.max_connections.saturating_add(self.soft_connections) { + self.active.insert(request.playback_id.clone(), request); + return Ok((Decision::AdmitSoft, None)); + } + for strategy in &self.strategies { + if let Some(mode) = grace_mode(*strategy) { + return Ok((Decision::Grace { mode }, None)); + } + let candidate = self.select(&request, *strategy)?; + if let Some(candidate) = candidate { + let victim = candidate.playback_id.clone(); + let evicted_req = self.active.remove(&victim); + self.active.insert(request.playback_id.clone(), request); + return Ok((Decision::Evict { playback_id: victim }, evicted_req)); + } + } + Ok(( + match self.grace { + Some(mode) => Decision::Grace { mode }, + None => Decision::Deny, + }, + None, + )) + } + + pub fn admit(&mut self, request: Request) -> Result { + self.admit_with_evicted(request).map(|(decision, _)| decision) + } + + pub fn rollback(&mut self, playback_id: &str, evicted: Option) { + self.active.remove(playback_id); + if let Some(restored) = evicted { + self.active.insert(restored.playback_id.clone(), restored); + } + } + + pub fn release(&mut self, playback_id: &str) -> Result<(), TestkitError> { + self.active + .remove(playback_id) + .map(|_| ()) + .ok_or_else(|| TestkitError::Configuration(format!("cannot release inactive playback {playback_id}"))) + } + + fn select(&self, incoming: &Request, strategy: AdmissionStrategy) -> Result, TestkitError> { + if grace_mode(strategy).is_some() { + return Ok(None); + } + let same_ip = + matches!(strategy, AdmissionStrategy::EvictUserSameIpOldest | AdmissionStrategy::EvictUserSameIpLatest); + let latest = matches!(strategy, AdmissionStrategy::EvictUserSameIpLatest | AdmissionStrategy::EvictUserLatest); + let candidates = self.active.values().filter(|candidate| !same_ip || candidate.client_ip == incoming.client_ip); + let selected = if latest { + candidates.max_by_key(|candidate| candidate.started_order) + } else { + candidates.min_by_key(|candidate| candidate.started_order) + }; + let Some(selected) = selected else { return Ok(None) }; + let tied = self.active.values().any(|candidate| { + candidate.playback_id != selected.playback_id + && (!same_ip || candidate.client_ip == incoming.client_ip) + && candidate.started_order == selected.started_order + }); + if tied { + return Err(TestkitError::Protocol("candidate ages are ambiguous at SUT timestamp resolution".to_owned())); + } + Ok(Some(selected)) + } +} + +const fn grace_mode(strategy: AdmissionStrategy) -> Option { + match strategy { + AdmissionStrategy::GraceInstantStream => Some(GraceMode::Instant), + AdmissionStrategy::GraceHoldStream => Some(GraceMode::HoldStream), + AdmissionStrategy::EvictUserSameIpOldest + | AdmissionStrategy::EvictUserSameIpLatest + | AdmissionStrategy::EvictUserOldest + | AdmissionStrategy::EvictUserLatest => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn latest_replaces_latest_candidate() { + let mut oracle = AdmissionOracle::new(2, vec![AdmissionStrategy::EvictUserSameIpLatest]); + for (id, order) in [("a", 1), ("b", 2)] { + assert!(matches!( + oracle.admit(Request { + playback_id: id.to_owned(), + client_ip: "10.0.0.1".to_owned(), + started_order: order + }), + Ok(Decision::Admit) + )); + } + assert!( + matches!(oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 3 }), Ok(Decision::Evict { playback_id }) if playback_id == "b") + ); + } + + #[test] + fn soft_slot_precedes_eviction() { + let mut oracle = + AdmissionOracle::with_policy(1, 1, vec![AdmissionStrategy::EvictUserLatest], Some(GraceMode::HoldStream)); + assert!(matches!( + oracle.admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }), + Ok(Decision::Admit) + )); + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }), + Ok(Decision::AdmitSoft) + )); + assert!( + matches!(oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.3".to_owned(), started_order: 3 }), Ok(Decision::Evict { playback_id }) if playback_id == "b") + ); + oracle.release("a").unwrap(); + oracle.release("c").unwrap(); + assert!(matches!( + oracle.admit(Request { playback_id: "d".to_owned(), client_ip: "10.0.0.4".to_owned(), started_order: 4 }), + Ok(Decision::Admit) + )); + assert!(matches!( + oracle.admit(Request { playback_id: "e".to_owned(), client_ip: "10.0.0.5".to_owned(), started_order: 5 }), + Ok(Decision::AdmitSoft) + )); + } + + #[test] + fn grace_applies_when_capacity_has_no_eviction_strategy() { + let mut oracle = AdmissionOracle::with_policy(1, 0, Vec::new(), Some(GraceMode::HoldStream)); + assert!(oracle + .admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }) + .is_ok()); + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }), + Ok(Decision::Grace { mode: GraceMode::HoldStream }) + )); + } + + #[test] + fn grace_strategy_is_evaluated_in_its_configured_order() { + let mut oracle = AdmissionOracle::new(1, vec![AdmissionStrategy::GraceInstantStream]); + assert!(oracle + .admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }) + .is_ok()); + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }), + Ok(Decision::Grace { mode: GraceMode::Instant }) + )); + } + + #[test] + fn equal_candidate_ages_are_ambiguous() { + let mut oracle = AdmissionOracle::new(2, vec![AdmissionStrategy::EvictUserOldest]); + for id in ["a", "b"] { + assert!(oracle + .admit(Request { playback_id: id.to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }) + .is_ok()); + } + assert!(oracle + .admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }) + .is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/origin_events.rs b/tools/tuliprox-testkit/src/origin_events.rs new file mode 100644 index 000000000..e60aa9db9 --- /dev/null +++ b/tools/tuliprox-testkit/src/origin_events.rs @@ -0,0 +1,127 @@ +use serde::{Deserialize, Serialize}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BodyCloseReason { + Completed, + ClientDisconnected, + Evicted, + Stalled, + Error(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum OriginEventKind { + TcpAccepted { + conn_id: u64, + remote_addr: String, + }, + RequestStarted { + conn_id: u64, + request_id: u64, + method: String, + path: String, + range: Option, + user_agent: Option, + account: Option, + }, + BodyStarted { + conn_id: u64, + request_id: u64, + content_length: Option, + }, + BodyClosed { + conn_id: u64, + request_id: u64, + bytes_emitted: u64, + duration_ms: u64, + reason: BodyCloseReason, + }, + TcpClosed { + conn_id: u64, + duration_ms: u64, + bytes_read: u64, + bytes_written: u64, + reason: String, + }, + LimitRejected { + conn_id: u64, + request_id: u64, + limit: usize, + current_active: usize, + }, + EvictionTriggered { + evicted_conn_id: u64, + evicted_request_id: u64, + triggering_conn_id: u64, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OriginEvent { + pub seq: u64, + pub timestamp_ns: u128, + pub run_id: String, + #[serde(flatten)] + pub kind: OriginEventKind, +} + +impl OriginEvent { + #[must_use] + pub fn new(seq: u64, run_id: impl Into, kind: OriginEventKind) -> Self { + let timestamp_ns = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |duration| duration.as_nanos()); + Self { seq, timestamp_ns, run_id: run_id.into(), kind } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct OriginStats { + pub active_tcp_connections: usize, + pub max_active_tcp_connections: usize, + pub active_body_connections: usize, + pub max_active_body_connections: usize, + pub total_tcp_connections: u64, + pub total_requests: u64, + pub total_bytes_emitted: u64, +} + +#[must_use] +pub fn redact_path_and_query(raw: &str) -> String { + let Some((path, query)) = raw.split_once('?') else { + return raw.to_owned(); + }; + let redacted_query: Vec = query + .split('&') + .map(|pair| { + if let Some((key, _)) = pair.split_once('=') { + let lower = key.to_ascii_lowercase(); + if lower.contains("pass") + || lower.contains("token") + || lower.contains("secret") + || lower.contains("key") + { + return format!("{key}="); + } + } + pair.to_owned() + }) + .collect(); + format!("{path}?{}", redacted_query.join("&")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redacts_sensitive_query_parameters() { + let path = "/movie/stream.mkv?user=alice&password=secret123&token=abc&run=test-run"; + let redacted = redact_path_and_query(path); + assert!(redacted.contains("user=alice")); + assert!(redacted.contains("password=")); + assert!(redacted.contains("token=")); + assert!(redacted.contains("run=test-run")); + } +} diff --git a/tools/tuliprox-testkit/src/origin_transport.rs b/tools/tuliprox-testkit/src/origin_transport.rs new file mode 100644 index 000000000..a176d6d63 --- /dev/null +++ b/tools/tuliprox-testkit/src/origin_transport.rs @@ -0,0 +1,484 @@ +use crate::origin_events::{redact_path_and_query, BodyCloseReason, OriginEvent, OriginEventKind, OriginStats}; +use pin_project_lite::pin_project; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + net::SocketAddr, + pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, + task::{Context, Poll}, + time::Instant, +}; +use tokio::{ + io::{AsyncRead, AsyncWrite, ReadBuf}, + sync::{oneshot, Mutex}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OriginLimitMode { + ObserveOnly, + RejectNew, + EvictOldest, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OriginPolicy { + pub account_limit: Option, + pub limit_mode: OriginLimitMode, +} + +impl Default for OriginPolicy { + fn default() -> Self { Self { account_limit: None, limit_mode: OriginLimitMode::ObserveOnly } } +} + +#[derive(Clone, Debug)] +pub struct OriginConnectionMeta { + pub conn_id: u64, + pub remote_addr: SocketAddr, +} + +struct ActiveConnection { + conn_id: u64, + #[allow(dead_code)] + remote_addr: SocketAddr, + connected_at: Instant, + active_body_req_id: Option, +} + +#[derive(Clone)] +pub struct OriginTracker { + run_id: String, + policy: Arc>, + events: Arc>>, + stats: Arc>, + active_conns: Arc>>, + eviction_channels: Arc>>>, + seq_counter: Arc, + conn_counter: Arc, + req_counter: Arc, +} + +impl OriginTracker { + #[must_use] + pub fn new(run_id: impl Into, policy: OriginPolicy) -> Self { + Self { + run_id: run_id.into(), + policy: Arc::new(Mutex::new(policy)), + events: Arc::new(Mutex::new(Vec::new())), + stats: Arc::new(Mutex::new(OriginStats::default())), + active_conns: Arc::new(Mutex::new(HashMap::new())), + eviction_channels: Arc::new(Mutex::new(HashMap::new())), + seq_counter: Arc::new(AtomicU64::new(0)), + conn_counter: Arc::new(AtomicU64::new(0)), + req_counter: Arc::new(AtomicU64::new(0)), + } + } + + fn next_seq(&self) -> u64 { self.seq_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1) } + + pub async fn set_policy(&self, policy: OriginPolicy) { *self.policy.lock().await = policy; } + + pub async fn policy(&self) -> OriginPolicy { self.policy.lock().await.clone() } + + pub async fn on_tcp_accepted(&self, remote_addr: SocketAddr) -> u64 { + let conn_id = self.conn_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_tcp_connections = stats.active_tcp_connections.saturating_add(1); + stats.total_tcp_connections = stats.total_tcp_connections.saturating_add(1); + if stats.active_tcp_connections > stats.max_active_tcp_connections { + stats.max_active_tcp_connections = stats.active_tcp_connections; + } + drop(stats); + + self.active_conns.lock().await.insert( + conn_id, + ActiveConnection { conn_id, remote_addr, connected_at: Instant::now(), active_body_req_id: None }, + ); + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::TcpAccepted { conn_id, remote_addr: remote_addr.to_string() }, + )); + + conn_id + } + + pub async fn on_request_started( + &self, + conn_id: u64, + method: &str, + path_and_query: &str, + range: Option<&str>, + user_agent: Option<&str>, + account: Option<&str>, + ) -> Result { + let req_id = self.req_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.total_requests = stats.total_requests.saturating_add(1); + let active_bodies = stats.active_body_connections; + drop(stats); + + let policy = self.policy.lock().await.clone(); + if let Some(limit) = policy.account_limit { + if active_bodies >= limit { + match policy.limit_mode { + OriginLimitMode::ObserveOnly => {} + OriginLimitMode::RejectNew => { + let reject_seq = self.next_seq(); + self.events.lock().await.push(OriginEvent::new( + reject_seq, + &self.run_id, + OriginEventKind::LimitRejected { + conn_id, + request_id: req_id, + limit, + current_active: active_bodies, + }, + )); + return Err((req_id, 429)); + } + OriginLimitMode::EvictOldest => { + self.evict_oldest_body(conn_id).await; + } + } + } + } + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::RequestStarted { + conn_id, + request_id: req_id, + method: method.to_owned(), + path: redact_path_and_query(path_and_query), + range: range.map(str::to_owned), + user_agent: user_agent.map(str::to_owned), + account: account.map(str::to_owned), + }, + )); + + Ok(req_id) + } + + async fn evict_oldest_body(&self, triggering_conn_id: u64) { + let conns = self.active_conns.lock().await; + let oldest_active = conns + .values() + .filter(|c| c.active_body_req_id.is_some() && c.conn_id != triggering_conn_id) + .min_by_key(|c| c.connected_at); + + if let Some(oldest) = oldest_active { + let evicted_conn_id = oldest.conn_id; + let evicted_req_id = oldest.active_body_req_id.unwrap_or(0); + drop(conns); + + if let Some(sender) = self.eviction_channels.lock().await.remove(&evicted_conn_id) { + let _ = sender.send(()); + } + + let seq = self.next_seq(); + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::EvictionTriggered { + evicted_conn_id, + evicted_request_id: evicted_req_id, + triggering_conn_id, + }, + )); + } + } + + pub async fn on_body_started( + &self, + conn_id: u64, + req_id: u64, + content_length: Option, + ) -> oneshot::Receiver<()> { + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_body_connections = stats.active_body_connections.saturating_add(1); + if stats.active_body_connections > stats.max_active_body_connections { + stats.max_active_body_connections = stats.active_body_connections; + } + drop(stats); + + if let Some(conn) = self.active_conns.lock().await.get_mut(&conn_id) { + conn.active_body_req_id = Some(req_id); + } + + let (tx, rx) = oneshot::channel(); + self.eviction_channels.lock().await.insert(conn_id, tx); + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::BodyStarted { conn_id, request_id: req_id, content_length }, + )); + + rx + } + + pub async fn on_body_closed( + &self, + conn_id: u64, + req_id: u64, + bytes_emitted: u64, + duration_ms: u64, + reason: BodyCloseReason, + ) { + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_body_connections = stats.active_body_connections.saturating_sub(1); + stats.total_bytes_emitted = stats.total_bytes_emitted.saturating_add(bytes_emitted); + drop(stats); + + self.eviction_channels.lock().await.remove(&conn_id); + if let Some(conn) = self.active_conns.lock().await.get_mut(&conn_id) { + if conn.active_body_req_id == Some(req_id) { + conn.active_body_req_id = None; + } + } + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::BodyClosed { conn_id, request_id: req_id, bytes_emitted, duration_ms, reason }, + )); + } + + pub async fn on_tcp_closed( + &self, + conn_id: u64, + duration_ms: u64, + bytes_read: u64, + bytes_written: u64, + reason: &str, + ) { + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_tcp_connections = stats.active_tcp_connections.saturating_sub(1); + drop(stats); + + self.active_conns.lock().await.remove(&conn_id); + self.eviction_channels.lock().await.remove(&conn_id); + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::TcpClosed { conn_id, duration_ms, bytes_read, bytes_written, reason: reason.to_owned() }, + )); + } + + pub async fn stats(&self) -> OriginStats { self.stats.lock().await.clone() } + + pub async fn events(&self) -> Vec { self.events.lock().await.clone() } + + pub async fn reset(&self) { + *self.stats.lock().await = OriginStats::default(); + self.events.lock().await.clear(); + self.active_conns.lock().await.clear(); + self.eviction_channels.lock().await.clear(); + self.seq_counter.store(0, Ordering::Relaxed); + self.conn_counter.store(0, Ordering::Relaxed); + self.req_counter.store(0, Ordering::Relaxed); + } +} + +struct TcpDropGuard { + conn_id: u64, + tracker: OriginTracker, + start_time: Instant, + bytes_read: Arc, + bytes_written: Arc, + closed: Arc, +} + +impl Drop for TcpDropGuard { + fn drop(&mut self) { + if !self.closed.swap(true, Ordering::SeqCst) { + let conn_id = self.conn_id; + let tracker = self.tracker.clone(); + let duration_ms = u64::try_from(self.start_time.elapsed().as_millis()).unwrap_or(u64::MAX); + let bytes_read = self.bytes_read.load(Ordering::Relaxed); + let bytes_written = self.bytes_written.load(Ordering::Relaxed); + tokio::spawn(async move { + tracker.on_tcp_closed(conn_id, duration_ms, bytes_read, bytes_written, "io_dropped").await; + }); + } + } +} + +pin_project! { + pub struct TrackedIo { + #[pin] + inner: T, + bytes_read: Arc, + bytes_written: Arc, + _guard: TcpDropGuard, + } +} + +impl TrackedIo { + pub fn new(inner: T, conn_id: u64, tracker: OriginTracker) -> (Self, Arc) { + let closed = Arc::new(AtomicBool::new(false)); + let bytes_read = Arc::new(AtomicU64::new(0)); + let bytes_written = Arc::new(AtomicU64::new(0)); + let guard = TcpDropGuard { + conn_id, + tracker, + start_time: Instant::now(), + bytes_read: bytes_read.clone(), + bytes_written: bytes_written.clone(), + closed: closed.clone(), + }; + (Self { inner, bytes_read, bytes_written, _guard: guard }, closed) + } +} + +impl AsyncRead for TrackedIo { + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let this = self.project(); + let prev_len = buf.filled().len(); + let poll_res = this.inner.poll_read(cx, buf); + if let Poll::Ready(Ok(())) = &poll_res { + let newly_read = buf.filled().len().saturating_sub(prev_len) as u64; + this.bytes_read.fetch_add(newly_read, Ordering::Relaxed); + } + poll_res + } +} + +impl AsyncWrite for TrackedIo { + fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + let this = self.project(); + let poll_res = this.inner.poll_write(cx, buf); + if let Poll::Ready(Ok(written)) = &poll_res { + this.bytes_written.fetch_add(*written as u64, Ordering::Relaxed); + } + poll_res + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_shutdown(cx) + } +} + +pub async fn serve_tracked_media_listener( + listener: tokio::net::TcpListener, + app: axum::Router, + tracker: OriginTracker, +) -> Result<(), std::io::Error> { + loop { + let (tcp_stream, remote_addr) = listener.accept().await?; + let conn_id = tracker.on_tcp_accepted(remote_addr).await; + let (tracked_io, _closed) = TrackedIo::new(tcp_stream, conn_id, tracker.clone()); + let io = hyper_util::rt::TokioIo::new(tracked_io); + let app = app.clone(); + let conn_meta = OriginConnectionMeta { conn_id, remote_addr }; + + tokio::spawn(async move { + let service = hyper::service::service_fn(move |req: hyper::Request| { + let mut req = req.map(axum::body::Body::new); + req.extensions_mut().insert(conn_meta.clone()); + let mut app = app.clone(); + async move { tower_service::Service::call(&mut app, req).await } + }); + + let builder = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); + let _ = builder.serve_connection(io, service).await; + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{IpAddr, Ipv4Addr}; + + #[tokio::test] + async fn tracking_records_tcp_lifecycle_and_peak_counters() { + let tracker = OriginTracker::new("test-run", OriginPolicy::default()); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); + let conn_id = tracker.on_tcp_accepted(addr).await; + assert_eq!(conn_id, 1); + + let stats = tracker.stats().await; + assert_eq!(stats.active_tcp_connections, 1); + assert_eq!(stats.max_active_tcp_connections, 1); + + let req_id = tracker + .on_request_started(conn_id, "GET", "/live/17.ts", None, Some("test/1"), None) + .await + .expect("request admitted"); + assert_eq!(req_id, 1); + + let _evict_rx = tracker.on_body_started(conn_id, req_id, None).await; + let stats = tracker.stats().await; + assert_eq!(stats.active_body_connections, 1); + assert_eq!(stats.max_active_body_connections, 1); + + tracker.on_body_closed(conn_id, req_id, 1024, 50, BodyCloseReason::Completed).await; + let stats = tracker.stats().await; + assert_eq!(stats.active_body_connections, 0); + assert_eq!(stats.max_active_body_connections, 1); + + tracker.on_tcp_closed(conn_id, 100, 200, 1024, "clean_close").await; + let stats = tracker.stats().await; + assert_eq!(stats.active_tcp_connections, 0); + assert_eq!(stats.max_active_tcp_connections, 1); + + let events = tracker.events().await; + assert_eq!(events.len(), 5); + } + + #[tokio::test] + async fn policy_rejects_new_when_account_limit_reached() { + let policy = OriginPolicy { account_limit: Some(1), limit_mode: OriginLimitMode::RejectNew }; + let tracker = OriginTracker::new("test-run", policy); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); + + let conn1 = tracker.on_tcp_accepted(addr).await; + let req1 = tracker.on_request_started(conn1, "GET", "/live/1.ts", None, None, None).await.unwrap(); + let _rx1 = tracker.on_body_started(conn1, req1, None).await; + + let conn2 = tracker.on_tcp_accepted(addr).await; + let req2_res = tracker.on_request_started(conn2, "GET", "/live/2.ts", None, None, None).await; + assert_eq!(req2_res, Err((2, 429))); + } + + #[tokio::test] + async fn policy_evicts_oldest_when_account_limit_reached() { + let policy = OriginPolicy { account_limit: Some(1), limit_mode: OriginLimitMode::EvictOldest }; + let tracker = OriginTracker::new("test-run", policy); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); + + let conn1 = tracker.on_tcp_accepted(addr).await; + let req1 = tracker.on_request_started(conn1, "GET", "/live/1.ts", None, None, None).await.unwrap(); + let mut rx1 = tracker.on_body_started(conn1, req1, None).await; + + let conn2 = tracker.on_tcp_accepted(addr).await; + let req2 = tracker.on_request_started(conn2, "GET", "/live/2.ts", None, None, None).await.unwrap(); + assert_eq!(req2, 2); + + // conn1 should have received eviction signal + assert!(rx1.try_recv().is_ok()); + } +} diff --git a/tools/tuliprox-testkit/src/policy.rs b/tools/tuliprox-testkit/src/policy.rs new file mode 100644 index 000000000..3dfa11d50 --- /dev/null +++ b/tools/tuliprox-testkit/src/policy.rs @@ -0,0 +1,265 @@ +use crate::{ + config::{GraceContract, PolicyContract}, + oracle::{AdmissionStrategy, GraceMode}, + TestkitError, +}; +use serde_json::Value; +use std::collections::HashMap; + +/// Validate the persisted fixture configuration before any playback is started. +/// +/// The v1 configuration endpoint intentionally returns configuration files, rather +/// than the atomically loaded runtime configuration. The controller therefore +/// uses this only for isolated fixtures it started itself; a remote instance needs +/// runtime evidence and is reported as inconclusive by the caller. +pub fn validate_fixture_policy(contract: &PolicyContract, config: &Value) -> Result<(), TestkitError> { + let configured_access_control = required_bool(config, "/config/user_access_control")?; + if configured_access_control != contract.user_access_control { + return mismatch("user_access_control", contract.user_access_control, configured_access_control); + } + + let stream = required_object(config, "/config/reverse_proxy/stream")?; + let configured_strategies = stream + .get("admission_strategies") + .map(|strategies| { + serde_json::from_value::>(strategies.clone()) + .map_err(|error| TestkitError::Protocol(format!("invalid fixture admission_strategies: {error}"))) + }) + .transpose()?; + if configured_strategies != contract.admission_strategies { + return Err(TestkitError::Configuration(format!( + "fixture admission_strategies mismatch: scenario {:?}, fixture {:?}", + contract.admission_strategies, configured_strategies + ))); + } + + if let Some(grace) = &contract.grace { + validate_grace(grace, stream, &contract.effective_admission_strategies())?; + } + + let configured_users = index_users(config)?; + for (username, expected) in &contract.users { + let configured = configured_users + .get(username.as_str()) + .ok_or_else(|| TestkitError::Configuration(format!("fixture policy does not contain user {username}")))?; + let max_connections = usize::try_from(required_u64(configured, "max_connections")?).map_err(|_| { + TestkitError::Protocol("fixture max_connections cannot be represented on this platform".to_owned()) + })?; + let soft_connections = usize::try_from(required_u64(configured, "soft_connections")?).map_err(|_| { + TestkitError::Protocol("fixture soft_connections cannot be represented on this platform".to_owned()) + })?; + if max_connections != expected.max_connections || soft_connections != expected.soft_connections { + return Err(TestkitError::Configuration(format!( + "fixture policy mismatch for user {username}: scenario max/soft {}/{}, fixture {}/{}", + expected.max_connections, expected.soft_connections, max_connections, soft_connections + ))); + } + } + + if let Some(expected_limit) = contract.provider_max_connections { + validate_provider_limit(expected_limit.into(), config)?; + } + + Ok(()) +} + +fn validate_provider_limit(expected_limit: usize, config: &Value) -> Result<(), TestkitError> { + let inputs = config + .pointer("/sources/inputs") + .and_then(Value::as_array) + .ok_or_else(|| TestkitError::Protocol("fixture configuration omits sources.inputs".to_owned()))?; + + let matching_input = inputs + .iter() + .find(|input| { + input.get("name").and_then(Value::as_str) == Some("testkit-origin") + || input.get("max_connections").is_some() + }) + .ok_or_else(|| { + TestkitError::Configuration( + "fixture sources do not contain a provider input with max_connections".to_owned(), + ) + })?; + + let configured_limit = matching_input + .get("max_connections") + .and_then(Value::as_u64) + .ok_or_else(|| TestkitError::Configuration("fixture provider input omits max_connections".to_owned()))?; + + let configured_limit = usize::try_from(configured_limit).map_err(|_| { + TestkitError::Protocol("fixture provider max_connections cannot be represented on this platform".to_owned()) + })?; + + if configured_limit != expected_limit { + return Err(TestkitError::Configuration(format!( + "fixture provider max_connections mismatch: scenario {expected_limit}, fixture {configured_limit}" + ))); + } + Ok(()) +} + +fn validate_grace( + expected: &GraceContract, + stream: &serde_json::Map, + strategies: &[AdmissionStrategy], +) -> Result<(), TestkitError> { + let timeout = required_u64_object(stream, "grace_period_millis")?; + if timeout != expected.timeout_millis { + return Err(TestkitError::Configuration(format!( + "fixture grace_period_millis mismatch: scenario {}, fixture {timeout}", + expected.timeout_millis + ))); + } + let hold = required_bool_object(stream, "grace_period_hold_stream")?; + let expected_hold = expected.mode == GraceMode::HoldStream; + if hold != expected_hold { + return mismatch("grace_period_hold_stream", expected_hold, hold); + } + // A grace mode without its corresponding strategy would be silently ignored + // by the SUT after capacity is exhausted. + let grace_strategy = match expected.mode { + GraceMode::Instant => "grace_instant_stream", + GraceMode::HoldStream => "grace_hold_stream", + }; + let encoded = serde_json::to_value(strategies) + .map_err(|error| TestkitError::Protocol(format!("cannot encode admission strategies: {error}")))?; + if !encoded.as_array().is_some_and(|values| values.iter().any(|value| value == grace_strategy)) { + return Err(TestkitError::Configuration(format!( + "fixture policy grace mode requires {grace_strategy} admission strategy" + ))); + } + Ok(()) +} + +fn index_users(config: &Value) -> Result, TestkitError> { + let users = config + .pointer("/api_proxy/user") + .and_then(Value::as_array) + .ok_or_else(|| TestkitError::Protocol("fixture configuration omits api_proxy.user".to_owned()))?; + let mut indexed = HashMap::new(); + for target in users { + let credentials = target + .get("credentials") + .and_then(Value::as_array) + .ok_or_else(|| TestkitError::Protocol("fixture api_proxy user entry omits credentials".to_owned()))?; + for user in credentials { + let username = user + .get("username") + .and_then(Value::as_str) + .ok_or_else(|| TestkitError::Protocol("fixture user omits username".to_owned()))?; + if indexed.insert(username, user).is_some() { + return Err(TestkitError::Configuration(format!("fixture has duplicate username {username}"))); + } + } + } + Ok(indexed) +} + +fn required_object<'a>(value: &'a Value, pointer: &str) -> Result<&'a serde_json::Map, TestkitError> { + value + .pointer(pointer) + .and_then(Value::as_object) + .ok_or_else(|| TestkitError::Protocol(format!("fixture configuration omits object {pointer}"))) +} + +fn required_bool(value: &Value, pointer: &str) -> Result { + value + .pointer(pointer) + .and_then(Value::as_bool) + .ok_or_else(|| TestkitError::Protocol(format!("fixture configuration omits boolean {pointer}"))) +} + +fn required_bool_object(value: &serde_json::Map, key: &str) -> Result { + value.get(key).and_then(Value::as_bool).ok_or_else(|| { + TestkitError::Protocol(format!("fixture configuration omits boolean config.reverse_proxy.stream.{key}")) + }) +} + +fn required_u64(value: &Value, key: &str) -> Result { + value + .get(key) + .and_then(Value::as_u64) + .ok_or_else(|| TestkitError::Protocol(format!("fixture user omits integer {key}"))) +} + +fn required_u64_object(value: &serde_json::Map, key: &str) -> Result { + value.get(key).and_then(Value::as_u64).ok_or_else(|| { + TestkitError::Protocol(format!("fixture configuration omits integer config.reverse_proxy.stream.{key}")) + }) +} + +fn mismatch(field: &str, expected: T, actual: T) -> Result<(), TestkitError> { + Err(TestkitError::Configuration(format!("fixture {field} mismatch: scenario {expected}, fixture {actual}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::UserPolicy; + use serde_json::json; + + fn contract() -> PolicyContract { + PolicyContract { + user_access_control: true, + users: HashMap::from([("alice".to_owned(), UserPolicy { max_connections: 2, soft_connections: 1 })]), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpLatest]), + grace: None, + provider_max_connections: None, + expected_provider_slots: None, + } + } + + fn fixture() -> Value { + json!({ + "config": {"user_access_control": true, "reverse_proxy": {"stream": {"admission_strategies": ["evict_user_same_ip_latest"]}}}, + "api_proxy": {"user": [{"credentials": [{"username": "alice", "max_connections": 2, "soft_connections": 1}]}]} + }) + } + + #[test] + fn accepts_matching_fixture_policy() { + assert!(validate_fixture_policy(&contract(), &fixture()).is_ok()); + } + + #[test] + fn rejects_mismatched_user_limit() { + let mut config = fixture(); + config["api_proxy"]["user"][0]["credentials"][0]["max_connections"] = json!(3); + assert!(validate_fixture_policy(&contract(), &config).is_err()); + } + + #[test] + fn distinguishes_omitted_and_explicit_admission_strategies() { + let mut config = fixture(); + config["config"]["reverse_proxy"]["stream"].as_object_mut().unwrap().remove("admission_strategies"); + assert!(matches!(validate_fixture_policy(&contract(), &config), Err(TestkitError::Configuration(_)))); + + let mut omitted_contract = contract(); + omitted_contract.admission_strategies = None; + assert!(validate_fixture_policy(&omitted_contract, &config).is_ok()); + } + + #[test] + fn validates_provider_max_connections() { + let mut contract_with_provider = contract(); + contract_with_provider.provider_max_connections = Some(2); + + let mut config_matching = fixture(); + config_matching["sources"] = json!({ + "inputs": [{"name": "testkit-origin", "max_connections": 2}] + }); + assert!(validate_fixture_policy(&contract_with_provider, &config_matching).is_ok()); + + let mut config_mismatched = fixture(); + config_mismatched["sources"] = json!({ + "inputs": [{"name": "testkit-origin", "max_connections": 1}] + }); + assert!(validate_fixture_policy(&contract_with_provider, &config_mismatched).is_err()); + + let mut config_missing = fixture(); + config_missing["sources"] = json!({ + "inputs": [{"name": "other-origin"}] + }); + assert!(validate_fixture_policy(&contract_with_provider, &config_missing).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/protocol.rs b/tools/tuliprox-testkit/src/protocol.rs new file mode 100644 index 000000000..e45d7747a --- /dev/null +++ b/tools/tuliprox-testkit/src/protocol.rs @@ -0,0 +1,235 @@ +use crate::TestkitError; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +pub const SCHEMA_VERSION: u16 = 1; + +macro_rules! id_type { + ($name:ident) => { + #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[serde(transparent)] + pub struct $name(pub String); + + impl $name { + #[must_use] + pub fn new(value: impl Into) -> Self { Self(value.into()) } + } + }; +} + +id_type!(RunId); +id_type!(AgentId); +id_type!(ActorId); +id_type!(PlaybackId); +id_type!(CommandId); +id_type!(OriginStreamId); + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Envelope { + pub schema_version: u16, + pub run_id: RunId, + pub run_generation: u64, + pub message_id: String, + pub agent_id: AgentId, + pub agent_boot_id: String, + pub source_sequence: u64, + pub caused_by_command_id: Option, + pub local_elapsed_nanos: u64, + pub payload: T, +} + +impl Envelope { + pub fn validate(&self) -> Result<(), TestkitError> { + if self.schema_version != SCHEMA_VERSION { + return Err(TestkitError::Protocol(format!("unsupported schema version {}", self.schema_version))); + } + if self.run_id.0.is_empty() || self.message_id.is_empty() || self.agent_id.0.is_empty() { + return Err(TestkitError::Protocol("empty envelope identity".to_owned())); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum Command { + ConfigureRun { + lease_millis: u64, + }, + StartPlayback { + playback_id: PlaybackId, + url: String, + #[serde(default)] + headers: std::collections::BTreeMap, + #[serde(default)] + expected_run_id: Option, + #[serde(default)] + expected_marker: Option, + #[serde(default)] + required_frames: Option, + }, + StopPlayback { + playback_id: PlaybackId, + }, + StopAll, + RenewRunLease { + lease_millis: u64, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RejectionReason { + CustomVideo(crate::custom_video::CustomVideoKind), + HttpStatus(u16), + ClosedBeforeFirstByte, + Other(String), +} + +impl std::fmt::Display for RejectionReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::CustomVideo(kind) => write!(f, "custom-video({kind:?})"), + Self::HttpStatus(status) => write!(f, "http-status({status})"), + Self::ClosedBeforeFirstByte => write!(f, "closed-before-first-byte"), + Self::Other(msg) => write!(f, "{msg}"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum PlaybackOutcome { + Streaming { frames: u64, bytes: u64 }, + AdmissionRejected { reason: RejectionReason }, + HttpError { status: u16 }, + TransportError { message: String }, + InvalidData { message: String }, + UnexpectedEof { frames: u64, bytes: u64 }, + IdleTimeout, + ExplicitStop, + InfrastructureError { message: String }, +} + +impl PlaybackOutcome { + #[must_use] + pub fn is_streaming(&self) -> bool { matches!(self, Self::Streaming { .. }) } + + #[must_use] + pub fn is_rejected(&self) -> bool { matches!(self, Self::AdmissionRejected { .. }) } +} + +impl std::fmt::Display for PlaybackOutcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Streaming { frames, bytes } => write!(f, "streaming({frames} frames, {bytes} bytes)"), + Self::AdmissionRejected { reason } => write!(f, "rejected({reason})"), + Self::HttpError { status } => write!(f, "http_error({status})"), + Self::TransportError { message } => write!(f, "transport_error({message})"), + Self::InvalidData { message } => write!(f, "invalid_data({message})"), + Self::UnexpectedEof { frames, bytes } => write!(f, "unexpected_eof({frames} frames, {bytes} bytes)"), + Self::IdleTimeout => write!(f, "idle_timeout"), + Self::ExplicitStop => write!(f, "explicit_stop"), + Self::InfrastructureError { message } => write!(f, "infrastructure_error({message})"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum PlaybackEvent { + CommandAccepted { + command_id: CommandId, + }, + HeadersReceived { + playback_id: PlaybackId, + status: u16, + }, + FirstValidFrame { + playback_id: PlaybackId, + sequence: u64, + }, + Progress { + playback_id: PlaybackId, + frames: u64, + bytes: u64, + }, + Terminal { + playback_id: PlaybackId, + outcome: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + typed_outcome: Option, + }, + LeaseExpired, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum AgentMessage { + Hello { hostname: String, supported_protocols: Vec, maximum_listeners: u32 }, + Ready, + Event { event: PlaybackEvent }, + Heartbeat, +} + +#[derive(Debug, Clone)] +pub struct RunLease { + generation: u64, + expires_at: tokio::time::Instant, +} + +impl RunLease { + #[must_use] + pub fn new(generation: u64, duration: Duration) -> Self { + Self { generation, expires_at: tokio::time::Instant::now() + duration } + } + + pub fn renew(&mut self, generation: u64, duration: Duration) -> Result<(), TestkitError> { + if generation != self.generation { + return Err(TestkitError::Protocol("stale run generation".to_owned())); + } + self.expires_at = tokio::time::Instant::now() + duration; + Ok(()) + } + + #[must_use] + pub fn expired(&self) -> bool { tokio::time::Instant::now() >= self.expires_at } + + #[must_use] + pub const fn deadline(&self) -> tokio::time::Instant { self.expires_at } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_unknown_schema() { + let envelope = Envelope { + schema_version: 2, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "message".to_owned(), + agent_id: AgentId::new("agent"), + agent_boot_id: "boot".to_owned(), + source_sequence: 0, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: Command::StopAll, + }; + assert!(envelope.validate().is_err()); + } + + #[tokio::test(start_paused = true)] + async fn lease_expiry_and_renewal_use_monotonic_time() { + let mut lease = RunLease::new(1, Duration::from_secs(5)); + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!lease.expired()); + lease.renew(1, Duration::from_secs(5)).unwrap(); + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!lease.expired()); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(lease.expired()); + } +} diff --git a/tools/tuliprox-testkit/src/report.rs b/tools/tuliprox-testkit/src/report.rs new file mode 100644 index 000000000..da5cae0b2 --- /dev/null +++ b/tools/tuliprox-testkit/src/report.rs @@ -0,0 +1,146 @@ +use crate::TestkitError; +use serde::Serialize; +use std::{fs, path::Path}; + +#[derive(Debug, Serialize, serde::Deserialize)] +pub struct RunReport<'a> { + pub schema_version: u16, + pub scenario: &'a str, + pub outcome: &'a str, + pub policy_hash: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub run_id: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_kind: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_message: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub exit_code: Option, + pub events: Vec>, +} + +#[derive(Debug, Serialize, serde::Deserialize)] +pub struct ReportEvent<'a> { + pub source: &'a str, + pub playback_id: &'a str, + pub event: &'a str, + pub detail: &'a str, +} + +impl RunReport<'_> { + pub fn write_to(&self, directory: &Path) -> Result<(), TestkitError> { + fs::create_dir_all(directory)?; + let json = serde_json::to_vec_pretty(self) + .map_err(|error| TestkitError::Protocol(format!("cannot serialize report: {error}")))?; + fs::write(directory.join("run.json"), json)?; + fs::write(directory.join("events.ndjson"), self.events_ndjson()?)?; + let run_id_line = self.run_id.map_or_else(String::new, |id| format!("run_id: {id}\n")); + let summary = format!( + "scenario: {}\noutcome: {}\npolicy: {}\n{}events: {}\n", + self.scenario, + self.outcome, + self.policy_hash, + run_id_line, + self.events.len() + ); + fs::write(directory.join("summary.txt"), summary)?; + fs::write(directory.join("junit.xml"), self.junit_xml())?; + Ok(()) + } + + fn junit_xml(&self) -> String { + let failed = usize::from(self.outcome != "passed"); + let name = xml_escape(self.scenario); + if failed == 0 { + format!( + "\n\n" + ) + } else { + format!( + "\n\n", + xml_escape(self.outcome) + ) + } + } + + fn events_ndjson(&self) -> Result { + let mut lines = String::new(); + for event in &self.events { + let encoded = serde_json::to_string(event) + .map_err(|error| TestkitError::Protocol(format!("cannot serialize report event: {error}")))?; + lines.push_str(&encoded); + lines.push('\n'); + } + Ok(lines) + } +} + +fn xml_escape(value: &str) -> String { + value.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn junit_marks_failed_runs() { + let report = RunReport { + schema_version: 1, + scenario: "a&b", + outcome: "failed", + policy_hash: "hash", + run_id: None, + error_kind: None, + error_message: None, + exit_code: None, + events: Vec::new(), + }; + assert!(report.junit_xml().contains("failures=\"1\"")); + assert!(report.junit_xml().contains("a&b")); + } + + #[test] + fn events_are_emitted_as_one_json_object_per_line() { + let report = RunReport { + schema_version: 1, + scenario: "scenario", + outcome: "passed", + policy_hash: "hash", + run_id: None, + error_kind: None, + error_message: None, + exit_code: None, + events: vec![ReportEvent { + source: "controller", + playback_id: "playback-a", + event: "started", + detail: "ok", + }], + }; + let lines = report.events_ndjson().unwrap(); + assert_eq!(lines.lines().count(), 1); + assert_eq!(serde_json::from_str::(lines.trim()).unwrap()["playback_id"], "playback-a"); + } + + #[test] + fn report_serializes_and_deserializes_with_run_id_and_error() { + let report = RunReport { + schema_version: 1, + scenario: "scenario_fail", + outcome: "failed", + policy_hash: "pre-start", + run_id: Some("test-run-123"), + error_kind: Some("bootstrap_error"), + error_message: Some("address already in use"), + exit_code: Some(4), + events: Vec::new(), + }; + let json = serde_json::to_string(&report).unwrap(); + let value: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(value["run_id"], "test-run-123"); + assert_eq!(value["error_kind"], "bootstrap_error"); + assert_eq!(value["error_message"], "address already in use"); + assert_eq!(value["exit_code"], 4); + } +} diff --git a/tools/tuliprox-testkit/src/scheduler.rs b/tools/tuliprox-testkit/src/scheduler.rs new file mode 100644 index 000000000..1169ce0b0 --- /dev/null +++ b/tools/tuliprox-testkit/src/scheduler.rs @@ -0,0 +1,85 @@ +use crate::TestkitError; +use std::{collections::HashMap, time::Duration}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct EventKey { + pub command_id: String, + pub event: String, +} + +#[derive(Debug, Default)] +pub struct Timeline { + anchors: HashMap, +} + +impl Timeline { + pub fn record(&mut self, key: EventKey) -> Result<(), TestkitError> { + if self.anchors.contains_key(&key) { + return Err(TestkitError::Protocol(format!("duplicate causal event {}:{}", key.command_id, key.event))); + } + self.anchors.insert(key, tokio::time::Instant::now()); + Ok(()) + } + + pub fn deadline(&self, key: &EventKey, within: Duration) -> Result { + self.anchors.get(key).copied().map(|anchor| anchor + within).ok_or_else(|| { + TestkitError::Configuration(format!("assertion anchor {}:{} was never armed", key.command_id, key.event)) + }) + } + + pub fn require_before( + &self, + key: &EventKey, + within: Duration, + observed_at: tokio::time::Instant, + ) -> Result<(), TestkitError> { + if observed_at > self.deadline(key, within)? { + return Err(TestkitError::Protocol(format!( + "event exceeded deadline anchored at {}:{}", + key.command_id, key.event + ))); + } + Ok(()) + } +} + +#[derive(Debug)] +pub struct ContinuousProgress { + max_gap: Duration, + last_frame: Option, +} + +impl ContinuousProgress { + #[must_use] + pub fn new(max_gap: Duration) -> Self { Self { max_gap, last_frame: None } } + + pub fn frame(&mut self, now: tokio::time::Instant) -> Result<(), TestkitError> { + if self.last_frame.is_some_and(|previous| now.duration_since(previous) > self.max_gap) { + return Err(TestkitError::Protocol("stream progress gap exceeded assertion bound".to_owned())); + } + self.last_frame = Some(now); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn deadline_remains_anchored_when_later_events_arrive() { + let mut timeline = Timeline::default(); + let key = EventKey { command_id: "start-b".to_owned(), event: "sent".to_owned() }; + timeline.record(key.clone()).unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(timeline.require_before(&key, Duration::from_secs(1), tokio::time::Instant::now()).is_err()); + } + + #[tokio::test(start_paused = true)] + async fn continuous_progress_rejects_long_gaps() { + let mut progress = ContinuousProgress::new(Duration::from_secs(1)); + progress.frame(tokio::time::Instant::now()).unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(progress.frame(tokio::time::Instant::now()).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/secret.rs b/tools/tuliprox-testkit/src/secret.rs new file mode 100644 index 000000000..77e532f34 --- /dev/null +++ b/tools/tuliprox-testkit/src/secret.rs @@ -0,0 +1,20 @@ +use crate::TestkitError; + +pub fn environment(name: &str) -> Result { + if name.trim().is_empty() { + return Err(TestkitError::Configuration("secret environment variable name is empty".to_owned())); + } + std::env::var(name) + .map_err(|_| TestkitError::Configuration(format!("required secret environment variable {name} is not set"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_variable_does_not_expose_a_value() { + let error = environment("TULIPROX_TESTKIT_MISSING_SECRET").unwrap_err().to_string(); + assert!(error.contains("TULIPROX_TESTKIT_MISSING_SECRET")); + } +} diff --git a/tools/tuliprox-testkit/src/shared.rs b/tools/tuliprox-testkit/src/shared.rs new file mode 100644 index 000000000..5f8f43fce --- /dev/null +++ b/tools/tuliprox-testkit/src/shared.rs @@ -0,0 +1,53 @@ +use crate::TestkitError; +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Default)] +pub struct SharedStreamModel { + subscribers: HashMap>, +} + +impl SharedStreamModel { + /// Returns true only when this subscriber causes a new origin connection. + pub fn subscribe(&mut self, marker: u32, playback_id: String) -> Result { + let subscribers = self.subscribers.entry(marker).or_default(); + if !subscribers.insert(playback_id) { + return Err(TestkitError::Protocol("shared-stream subscriber was registered twice".to_owned())); + } + Ok(subscribers.len() == 1) + } + + /// Returns true only when the last subscriber closes the origin connection. + pub fn unsubscribe(&mut self, marker: u32, playback_id: &str) -> Result { + let subscribers = self + .subscribers + .get_mut(&marker) + .ok_or_else(|| TestkitError::Configuration(format!("shared stream marker {marker} is not active")))?; + if !subscribers.remove(playback_id) { + return Err(TestkitError::Configuration(format!( + "playback {playback_id} is not a shared-stream subscriber" + ))); + } + let last = subscribers.is_empty(); + if last { + self.subscribers.remove(&marker); + } + Ok(last) + } + + #[must_use] + pub fn subscriber_count(&self, marker: u32) -> usize { self.subscribers.get(&marker).map_or(0, HashSet::len) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_first_and_last_subscriber_change_origin_lifetime() { + let mut model = SharedStreamModel::default(); + assert!(model.subscribe(17, "a".to_owned()).unwrap()); + assert!(!model.subscribe(17, "b".to_owned()).unwrap()); + assert!(!model.unsubscribe(17, "a").unwrap()); + assert!(model.unsubscribe(17, "b").unwrap()); + } +} diff --git a/tools/tuliprox-testkit/src/transport.rs b/tools/tuliprox-testkit/src/transport.rs new file mode 100644 index 000000000..d5b68fe15 --- /dev/null +++ b/tools/tuliprox-testkit/src/transport.rs @@ -0,0 +1,372 @@ +use crate::{ + protocol::{AgentId, AgentMessage, Command, Envelope, PlaybackEvent, RunId}, + TestkitError, +}; +use axum::{ + extract::{ + ws::{Message, WebSocket}, + State, WebSocketUpgrade, + }, + response::{IntoResponse, Response}, +}; +use futures::{SinkExt, StreamExt}; +use std::{ + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + sync::{mpsc, Mutex}, + time::Instant, +}; +use tokio_tungstenite::{connect_async, tungstenite::protocol::Message as ClientMessage}; + +pub type ControllerState = Arc; + +pub struct SharedControllerState { + run_id: RunId, + generation: u64, + agents: Mutex>>>, + ready_agents: Mutex>, + last_seen: Mutex>, + events: Mutex>>, + next_controller_sequence: AtomicU64, +} + +impl SharedControllerState { + #[must_use] + pub fn new(run_id: RunId, generation: u64) -> Self { + Self { + run_id, + generation, + agents: Mutex::new(HashMap::new()), + ready_agents: Mutex::new(HashSet::new()), + last_seen: Mutex::new(HashMap::new()), + events: Mutex::new(Vec::new()), + next_controller_sequence: AtomicU64::new(1), + } + } + + #[must_use] + pub fn next_command_sequence(&self) -> u64 { self.next_controller_sequence.fetch_add(1, Ordering::Relaxed) } + + #[must_use] + pub fn run_id(&self) -> RunId { self.run_id.clone() } + + pub async fn dispatch(&self, agent_id: &AgentId, command: Envelope) -> Result<(), TestkitError> { + command.validate()?; + if command.run_id != self.run_id || command.run_generation != self.generation { + return Err(TestkitError::Protocol("command does not belong to the active run generation".to_owned())); + } + let sender = self + .agents + .lock() + .await + .get(agent_id) + .cloned() + .ok_or_else(|| TestkitError::Protocol(format!("agent {} is not connected", agent_id.0)))?; + sender.send(command).await.map_err(|_| TestkitError::Protocol("agent control channel closed".to_owned())) + } + + pub async fn connected_agents(&self) -> Vec { self.agents.lock().await.keys().cloned().collect() } + + pub async fn ready_agents(&self) -> Vec { + const HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(15); + let now = Instant::now(); + let ready = self.ready_agents.lock().await; + let seen = self.last_seen.lock().await; + ready + .iter() + .filter(|agent| { + seen.get(*agent).is_some_and(|last_seen| now.duration_since(*last_seen) <= HEARTBEAT_TIMEOUT) + }) + .cloned() + .collect() + } + + pub async fn take_events(&self) -> Vec> { std::mem::take(&mut *self.events.lock().await) } + + /// Removes only an event belonging to the requested playback. Events for + /// concurrent starts remain queued for their own waiter. + pub async fn take_playback_event(&self, playback_id: &str) -> Option { + let mut events = self.events.lock().await; + let position = events.iter().position(|envelope| { + matches!( + &envelope.payload, + AgentMessage::Event { + event: PlaybackEvent::FirstValidFrame { playback_id: observed, .. } + | PlaybackEvent::HeadersReceived { playback_id: observed, .. } + | PlaybackEvent::Progress { playback_id: observed, .. } + | PlaybackEvent::Terminal { playback_id: observed, .. }, + } if observed.0 == playback_id + ) + })?; + match events.remove(position).payload { + AgentMessage::Event { event } => Some(event), + AgentMessage::Hello { .. } | AgentMessage::Ready | AgentMessage::Heartbeat => None, + } + } +} + +pub async fn controller_websocket(State(state): State, websocket: WebSocketUpgrade) -> Response { + std::future::ready(()).await; + websocket.on_upgrade(move |socket| serve_agent(socket, state)).into_response() +} + +async fn serve_agent(socket: WebSocket, state: ControllerState) { + let (mut sink, mut stream) = socket.split(); + let Some(Ok(Message::Text(first))) = stream.next().await else { return }; + let Ok(hello) = serde_json::from_str::>(&first) else { return }; + if hello.validate().is_err() || hello.run_id != state.run_id || hello.run_generation != state.generation { + return; + } + let AgentMessage::Hello { .. } = hello.payload else { return }; + let agent_id = hello.agent_id; + let (tx, mut rx) = mpsc::channel(32); + state.agents.lock().await.insert(agent_id.clone(), tx); + state.last_seen.lock().await.insert(agent_id.clone(), Instant::now()); + loop { + tokio::select! { + Some(command) = rx.recv() => { + let Ok(encoded) = serde_json::to_string(&command) else { break }; + if sink.send(Message::Text(encoded.into())).await.is_err() { break; } + } + incoming = stream.next() => match incoming { + Some(Ok(Message::Close(_)) | Err(_)) | None => break, + Some(Ok(Message::Text(text))) => { + if let Ok(event) = serde_json::from_str::>(&text) { + if event.validate().is_ok() + && event.run_id == state.run_id + && event.run_generation == state.generation + && event.agent_id == agent_id + { + state.last_seen.lock().await.insert(agent_id.clone(), Instant::now()); + match event.payload { + AgentMessage::Ready => { + state.ready_agents.lock().await.insert(agent_id.clone()); + } + AgentMessage::Event { .. } => state.events.lock().await.push(event), + AgentMessage::Hello { .. } | AgentMessage::Heartbeat => {} + } + } + } + } + _ => {} + }, + } + } + state.agents.lock().await.remove(&agent_id); + state.ready_agents.lock().await.remove(&agent_id); + state.last_seen.lock().await.remove(&agent_id); +} + +pub struct AgentControlConnection { + commands: mpsc::Receiver>, + events: mpsc::Sender>, +} + +impl AgentControlConnection { + pub async fn connect(controller_url: &str, hello: Envelope) -> Result { + hello.validate()?; + if !matches!(hello.payload, AgentMessage::Hello { .. }) { + return Err(TestkitError::Protocol("first agent message must be hello".to_owned())); + } + let (mut socket, _) = + connect_async(controller_url).await.map_err(|error| TestkitError::Protocol(error.to_string()))?; + let encoded = serde_json::to_string(&hello).map_err(|error| TestkitError::Protocol(error.to_string()))?; + socket + .send(ClientMessage::Text(encoded.into())) + .await + .map_err(|error| TestkitError::Protocol(error.to_string()))?; + let (mut sink, mut stream) = socket.split(); + let (command_tx, command_rx) = mpsc::channel(32); + let (event_tx, mut event_rx) = mpsc::channel::>(32); + tokio::spawn(async move { + while let Some(event) = event_rx.recv().await { + let Ok(encoded) = serde_json::to_string(&event) else { break }; + if sink.send(ClientMessage::Text(encoded.into())).await.is_err() { + break; + } + } + }); + tokio::spawn(async move { + while let Some(message) = stream.next().await { + let Ok(message) = message else { break }; + match message { + ClientMessage::Text(text) => { + let Ok(command) = serde_json::from_str::>(&text) else { continue }; + if command.validate().is_err() || command_tx.send(command).await.is_err() { + break; + } + } + ClientMessage::Close(_) => break, + _ => {} + } + } + }); + Ok(Self { commands: command_rx, events: event_tx }) + } + + pub async fn next_command(&mut self) -> Result>, TestkitError> { + Ok(self.commands.recv().await) + } + + pub async fn send_event(&self, event: Envelope) -> Result<(), TestkitError> { + event.validate()?; + self.events.send(event).await.map_err(|_| TestkitError::Protocol("agent event channel closed".to_owned())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{routing::get, Router}; + + #[tokio::test] + async fn unknown_agent_cannot_receive_commands() { + let state = SharedControllerState::new(RunId::new("run"), 1); + let command = Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "command".to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: Command::StopAll, + }; + assert!(state.dispatch(&AgentId::new("missing"), command).await.is_err()); + } + + #[tokio::test] + async fn agent_receives_generation_bound_command_over_websocket() { + let state: ControllerState = Arc::new(SharedControllerState::new(RunId::new("run"), 1)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().route("/agent", get(controller_websocket)).with_state(state.clone()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let agent_id = AgentId::new("agent"); + let hello = Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "hello".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Hello { + hostname: "test".to_owned(), + supported_protocols: vec!["ts".to_owned()], + maximum_listeners: 1, + }, + }; + let mut agent = AgentControlConnection::connect(&format!("ws://{address}/agent"), hello).await.unwrap(); + for _ in 0..10 { + if !state.connected_agents().await.is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + agent + .send_event(Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "ready".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "boot".to_owned(), + source_sequence: 2, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Ready, + }) + .await + .unwrap(); + for _ in 0..10 { + if state.ready_agents().await.contains(&agent_id) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(state.ready_agents().await.contains(&agent_id)); + let command = Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "stop".to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: Command::StopAll, + }; + state.dispatch(&agent_id, command).await.unwrap(); + assert!(matches!(agent.next_command().await, Ok(Some(Envelope { payload: Command::StopAll, .. })))); + agent + .send_event(Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "event".to_owned(), + agent_id, + agent_boot_id: "boot".to_owned(), + source_sequence: 3, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Event { + event: crate::protocol::PlaybackEvent::Terminal { + playback_id: crate::protocol::PlaybackId::new("playback"), + outcome: "passed".to_owned(), + typed_outcome: None, + }, + }, + }) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + assert_eq!(state.take_events().await.len(), 1); + server.abort(); + } + + #[tokio::test] + async fn waiting_for_one_playback_keeps_other_playback_events() { + let state = SharedControllerState::new(RunId::new("run"), 1); + for playback_id in ["first", "second"] { + state.events.lock().await.push(Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: format!("event-{playback_id}"), + agent_id: AgentId::new("agent"), + agent_boot_id: "boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Event { + event: PlaybackEvent::FirstValidFrame { + playback_id: crate::protocol::PlaybackId::new(playback_id), + sequence: 0, + }, + }, + }); + } + assert!(matches!( + state.take_playback_event("second").await, + Some(PlaybackEvent::FirstValidFrame { playback_id, .. }) if playback_id.0 == "second" + )); + assert!(matches!( + state.take_playback_event("first").await, + Some(PlaybackEvent::FirstValidFrame { playback_id, .. }) if playback_id.0 == "first" + )); + } +} diff --git a/tools/tuliprox-testkit/src/vod.rs b/tools/tuliprox-testkit/src/vod.rs new file mode 100644 index 000000000..956fbd28b --- /dev/null +++ b/tools/tuliprox-testkit/src/vod.rs @@ -0,0 +1,345 @@ +use crate::TestkitError; +use bytes::Bytes; +use futures::Stream; +use reqwest::{header, StatusCode}; +use std::{pin::Pin, time::Duration}; +use tokio::sync::oneshot; + +pub const DEFAULT_VOD_SIZE: u64 = 128 * 1024 * 1024; // 128 MiB virtual media file +const BLOCK_SIZE: usize = 1024; + +pub async fn read_range(url: &str, start: u64, end_inclusive: u64) -> Result, TestkitError> { + if start > end_inclusive { + return Err(TestkitError::Configuration("VOD range start exceeds end".to_owned())); + } + let range = format!("bytes={start}-{end_inclusive}"); + let response = reqwest::Client::new().get(url).header(header::RANGE, range).send().await?; + if response.status() != StatusCode::PARTIAL_CONTENT { + return Err(TestkitError::Protocol(format!("VOD range returned {}, expected 206", response.status()))); + } + let expected_range = format!("bytes {start}-{end_inclusive}/"); + let content_range = response + .headers() + .get(header::CONTENT_RANGE) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| TestkitError::Protocol("VOD range response has no Content-Range".to_owned()))?; + if !content_range.starts_with(&expected_range) { + return Err(TestkitError::Protocol(format!("unexpected VOD Content-Range {content_range}"))); + } + let body = response.bytes().await?.to_vec(); + let expected_len = usize::try_from(end_inclusive.saturating_sub(start).saturating_add(1)) + .map_err(|_| TestkitError::Configuration("VOD range is too large for this host".to_owned()))?; + if body.len() != expected_len { + return Err(TestkitError::Protocol(format!("VOD range has {} bytes, expected {expected_len}", body.len()))); + } + Ok(body) +} + +#[must_use] +pub fn parse_range_header(value: &str, total_length: u64) -> Option<(u64, u64)> { + let bytes = value.strip_prefix("bytes=")?; + if let Some(suffix) = bytes.strip_prefix('-') { + let suffix_len = suffix.parse::().ok()?; + if suffix_len == 0 || suffix_len > total_length { + return Some((0, total_length)); + } + let start = total_length.saturating_sub(suffix_len); + return Some((start, total_length)); + } + + let (start_str, end_str) = bytes.split_once('-')?; + let start = start_str.parse::().ok()?; + if start >= total_length { + return None; + } + + let end = if end_str.is_empty() { + total_length + } else { + let parsed_end = end_str.parse::().ok()?; + parsed_end.min(total_length.saturating_sub(1)).saturating_add(1) + }; + + if start >= end { + return None; + } + + Some((start, end)) +} + +#[must_use] +pub fn parse_content_range(value: &str) -> Option<(u64, u64, Option)> { + let bytes = value.strip_prefix("bytes ")?; + let (range_part, total_part) = bytes.split_once('/')?; + let (start_str, end_str) = range_part.split_once('-')?; + let start = start_str.parse::().ok()?; + let end = end_str.parse::().ok()?; + let total = if total_part == "*" { None } else { Some(total_part.parse::().ok()?) }; + Some((start, end, total)) +} + +#[must_use] +pub fn generate_deterministic_block(object: &str, block_index: u64) -> [u8; BLOCK_SIZE] { + let seed = blake3::hash(object.as_bytes()); + let block_hash = blake3::keyed_hash(seed.as_bytes(), &block_index.to_le_bytes()); + let mut block = [0_u8; BLOCK_SIZE]; + block[0..8].copy_from_slice(&block_index.to_be_bytes()); + let hash_bytes = block_hash.as_bytes(); + for chunk in block[8..].chunks_mut(32) { + let len = chunk.len().min(32); + chunk.copy_from_slice(&hash_bytes[..len]); + } + block +} + +#[must_use] +pub fn generate_deterministic_slice(object: &str, offset: u64, len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let mut current_offset = offset; + let end_offset = offset.saturating_add(len as u64); + + while current_offset < end_offset { + let block_index = current_offset / (BLOCK_SIZE as u64); + let in_block = usize::try_from(current_offset % (BLOCK_SIZE as u64)).unwrap_or(0); + let block = generate_deterministic_block(object, block_index); + let remaining_in_block = BLOCK_SIZE.saturating_sub(in_block); + let remaining_requested = usize::try_from(end_offset.saturating_sub(current_offset)).unwrap_or(0); + let take = remaining_in_block.min(remaining_requested); + out.extend_from_slice(&block[in_block..in_block.saturating_add(take)]); + current_offset = current_offset.saturating_add(take as u64); + } + out +} + +#[must_use] +pub fn verify_deterministic_slice(object: &str, offset: u64, data: &[u8]) -> bool { + let expected = generate_deterministic_slice(object, offset, data.len()); + data == expected.as_slice() +} + +#[derive(Debug, Clone)] +pub struct VodStreamOptions { + pub object: String, + pub start_offset: u64, + pub length: u64, + pub bitrate: u32, + pub chunk_size: usize, + pub stall_ms: Option, + pub abort_after_bytes: Option, +} + +impl Default for VodStreamOptions { + fn default() -> Self { + Self { + object: "movie.mkv".to_owned(), + start_offset: 0, + length: DEFAULT_VOD_SIZE, + bitrate: 0, + chunk_size: 64 * 1024, + stall_ms: None, + abort_after_bytes: None, + } + } +} + +struct VodStreamState { + options: VodStreamOptions, + evict_rx: oneshot::Receiver<()>, + evicted_flag: Option>, + current_offset: u64, + end_offset: u64, + total_emitted: u64, + interval: Option, + initial_stall_done: bool, +} + +pub fn create_vod_stream( + options: VodStreamOptions, + evict_rx: oneshot::Receiver<()>, +) -> Pin> + Send>> { + create_vod_stream_with_evicted(options, evict_rx, None) +} + +pub fn create_vod_stream_with_evicted( + options: VodStreamOptions, + evict_rx: oneshot::Receiver<()>, + evicted_flag: Option>, +) -> Pin> + Send>> { + let current_offset = options.start_offset; + let end_offset = options.start_offset.saturating_add(options.length); + + let interval = if options.bitrate > 0 { + let bits_per_chunk = (options.chunk_size as u128).saturating_mul(8); + let nanos = + bits_per_chunk.saturating_mul(1_000_000_000).checked_div(u128::from(options.bitrate.max(1))).unwrap_or(1); + let duration = Duration::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX).max(1)); + let mut interval = tokio::time::interval(duration); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + Some(interval) + } else { + None + }; + + let state = VodStreamState { + options, + evict_rx, + evicted_flag, + current_offset, + end_offset, + total_emitted: 0, + interval, + initial_stall_done: false, + }; + + Box::pin(futures::stream::unfold(state, |mut state| async move { + if !state.initial_stall_done { + state.initial_stall_done = true; + if let Some(stall) = state.options.stall_ms { + tokio::time::sleep(Duration::from_millis(stall)).await; + } + } + + if state.current_offset >= state.end_offset { + return None; + } + + if let Some(abort_limit) = state.options.abort_after_bytes { + if state.total_emitted >= abort_limit { + return None; + } + } + + let remaining = usize::try_from(state.end_offset.saturating_sub(state.current_offset)).unwrap_or(0); + let mut take = state.options.chunk_size.min(remaining); + if let Some(abort_limit) = state.options.abort_after_bytes { + let bytes_until_abort = usize::try_from(abort_limit.saturating_sub(state.total_emitted)).unwrap_or(0); + take = take.min(bytes_until_abort); + } + + if take == 0 { + return None; + } + + if let Some(interval) = state.interval.as_mut() { + tokio::select! { + biased; + res = &mut state.evict_rx => { + if res.is_ok() { + if let Some(flag) = state.evicted_flag.as_ref() { + flag.store(true, std::sync::atomic::Ordering::Release); + } + } + return None; + } + _ = interval.tick() => {} + } + } else { + match state.evict_rx.try_recv() { + Ok(()) => { + if let Some(flag) = state.evicted_flag.as_ref() { + flag.store(true, std::sync::atomic::Ordering::Release); + } + return None; + } + Err(oneshot::error::TryRecvError::Closed) => return None, + Err(oneshot::error::TryRecvError::Empty) => {} + } + } + + let chunk = generate_deterministic_slice(&state.options.object, state.current_offset, take); + state.current_offset = state.current_offset.saturating_add(take as u64); + state.total_emitted = state.total_emitted.saturating_add(take as u64); + + Some((Ok(Bytes::from(chunk)), state)) + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio_stream::StreamExt; + + #[test] + fn parses_open_and_closed_and_suffix_ranges() { + assert_eq!(parse_range_header("bytes=0-1023", 10_000), Some((0, 1024))); + assert_eq!(parse_range_header("bytes=100-", 10_000), Some((100, 10_000))); + assert_eq!(parse_range_header("bytes=-500", 10_000), Some((9500, 10_000))); + assert_eq!(parse_range_header("bytes=10000-", 10_000), None); + assert_eq!(parse_range_header("bytes=500-200", 10_000), None); + } + + #[test] + fn deterministic_slice_verification_is_exact() { + let object = "test_vod.mkv"; + let offset = 4096; + let slice = generate_deterministic_slice(object, offset, 2048); + assert_eq!(slice.len(), 2048); + assert!(verify_deterministic_slice(object, offset, &slice)); + + // Tampered byte must fail verification + let mut tampered = slice.clone(); + tampered[10] ^= 0xFF; + assert!(!verify_deterministic_slice(object, offset, &tampered)); + + // Wrong offset must fail verification + assert!(!verify_deterministic_slice(object, offset + 1, &slice)); + } + + #[tokio::test] + async fn vod_stream_respects_abort_after_bytes() { + let options = VodStreamOptions { + object: "movie.mkv".to_owned(), + start_offset: 0, + length: 100_000, + bitrate: 0, + chunk_size: 1024, + stall_ms: None, + abort_after_bytes: Some(2500), + }; + let (_tx, rx) = oneshot::channel(); + let mut stream = create_vod_stream(options, rx); + + let mut collected = Vec::new(); + while let Some(Ok(chunk)) = stream.next().await { + collected.extend_from_slice(&chunk); + } + + assert_eq!(collected.len(), 2500); + assert!(verify_deterministic_slice("movie.mkv", 0, &collected)); + } + + #[tokio::test] + async fn vod_stream_stops_on_eviction() { + let options = VodStreamOptions { + object: "movie.mkv".to_owned(), + start_offset: 0, + length: 1_000_000, + bitrate: 0, + chunk_size: 1024, + stall_ms: None, + abort_after_bytes: None, + }; + let (tx, rx) = oneshot::channel(); + let mut stream = create_vod_stream(options, rx); + + // Read 1 chunk + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.len(), 1024); + + // Send eviction signal + let _ = tx.send(()); + + // Next read must terminate + assert!(stream.next().await.is_none()); + } + + #[test] + fn parse_content_range_valid_and_invalid() { + assert_eq!(parse_content_range("bytes 0-65535/134217728"), Some((0, 65535, Some(134_217_728)))); + assert_eq!(parse_content_range("bytes 100-200/*"), Some((100, 200, None))); + assert_eq!(parse_content_range("bytes 0-0/1"), Some((0, 0, Some(1)))); + assert_eq!(parse_content_range("0-65535/134217728"), None); + assert_eq!(parse_content_range("bytes 0-65535"), None); + assert_eq!(parse_content_range("bytes invalid-range/100"), None); + } +}