euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-19 14:22:19 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

test-kit

Browse Source
This commit is contained in:
euzu
2026-09-14 13:34:32 +02:00
parent bbd1297e28
commit 4ff3796727
58 changed files with 10129 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yaml
+40
View File
@@ -140,3 +140,43 @@ jobs:
- name: Run Tests
run: make test
admission:
name: Admission & Provider Lifecycle
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- name: Install Stable Rust
uses: dtolnay/rust-toolchain@stable
- name: Rust Cache
uses: Swatinem/rust-cache@v2
- name: Verify admission and provider lifecycle invariants
run: make admission-test
testkit:
name: Testkit Unit & E2E Scenarios
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- name: Install Stable Rust
uses: dtolnay/rust-toolchain@stable
- name: Rust Cache
uses: Swatinem/rust-cache@v2
- name: Run Testkit Unit Tests
run: make testkit-test
- name: Run Testkit E2E Scenarios
run: make testkit-e2e
- name: Upload Testkit Reports
if: always()
uses: actions/upload-artifact@v4
with:
name: testkit-report
path: testkit-report/
.gitignore
+3
View File
@@ -72,3 +72,6 @@ backend/app/Cargo.lock
.env.*
!.env.example
# Testkit report artifacts
/testkit-report/
Cargo.lock
Generated
+28
View File
@@ -4896,6 +4896,34 @@ dependencies = [
"url",
]
[[package]]
name = "tuliprox-testkit"
version = "3.3.109"
dependencies = [
"axum",
"base64 0.23.1",
"blake3",
"bytes",
"clap",
"crc32fast",
"futures",
"hyper",
"hyper-util",
"pin-project-lite",
"reqwest",
"serde",
"serde-saphyr",
"serde_json",
"tempfile",
"thiserror 2.0.20",
"tokio",
"tokio-stream",
"tokio-tungstenite",
"tower",
"tower-service",
"url",
]
[[package]]
name = "tungstenite"
version = "0.29.0"
Cargo.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[workspace]
members = ["backend/*", "frontend", "shared"]
members = ["backend/*", "frontend", "shared", "tools/tuliprox-testkit"]
resolver = "2"
[workspace.package]
Makefile
+23
View File
@@ -149,6 +149,27 @@ test: ## Run all workspace tests (Stable) — use detected CPU count for paralle
@echo "==> Running tests (stable) with $(CPU_COUNT) jobs/threads"
@TMPDIR="$${TMPDIR:-/tmp}" RUST_TEST_THREADS=$(CPU_COUNT) ./bin/test.sh -j$(CPU_COUNT) --workspace -- --test-threads=$(CPU_COUNT)
.PHONY: admission-test
admission-test: ## Verify admission, provider-slot, seek/reopen, shared-stream, and cleanup invariants
@echo "==> Running HTTP admission and provider lifecycle tests"
@$(CARGO_STABLE) test --package tuliprox api_utils::tests:: -- --test-threads=1
@echo "==> Running client stream lifecycle tests"
@$(CARGO_STABLE) test --package tuliprox api::model::streams::active_client_stream::tests:: -- --test-threads=1
@echo "==> Running session, eviction, and provider-slot lifecycle tests"
@$(CARGO_STABLE) test --package tuliprox-session -- --test-threads=1
.PHONY: testkit-test
testkit-test: ## Run unit tests in tuliprox-testkit
@echo "==> Running testkit unit tests"
@$(CARGO_STABLE) test --package tuliprox-testkit
.PHONY: testkit-e2e
testkit-e2e: ## Run testkit E2E scenario suite against freshly built SUT
@echo "==> Building tuliprox and tuliprox-testkit"
@$(CARGO_STABLE) build --package tuliprox --package tuliprox-testkit
@echo "==> Executing testkit scenario suite"
@./bin/run-testkit-scenarios.sh
.PHONY: build
build: ## Build the entire workspace in parallel using detected CPU count
@echo "==> Building workspace with $(CARGO_BUILD_JOBS) jobs"
@@ -180,6 +201,8 @@ fmt: ## Format all code using nightly rules (Compact)
fmt-check: ## Check if code follows formatting rules (Nightly)
@echo "==> Checking formatting (nightly)"
$(CARGO_NIGHTLY) fmt --all -- --check
@echo "==> Checking git diff for whitespace errors"
git diff --check
.PHONY: markdown-lint
markdown-lint: ## Lint markdown files
backend/app/src/api/api_utils/mod.rs
+67 -23
View File
@@ -7,12 +7,13 @@ use crate::{
endpoints::xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext},
model::{
create_active_client_stream, create_channel_unavailable_stream, create_custom_video_stream_response,
create_provider_connections_exhausted_stream, create_provider_stream,
get_custom_stream_response_error_status, get_stream_response_with_headers, is_custom_video_stream_enabled,
create_provider_connections_exhausted_stream, get_custom_stream_response_error_status,
get_stream_response_with_headers, is_custom_video_stream_enabled, open_provider_stream_with_lifecycle,
tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, PlaybackLeaseRef,
ProviderAllocation, ProviderConfig, ProviderStreamCustomReason, ProviderStreamFactoryOptions,
ProviderStreamInfo, ProviderStreamState, SharedStreamCtx, SharedStreamManager, StreamAdmissionError,
StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, UserSession,
ProviderStreamInfo, ProviderStreamOpenLifecycle, ProviderStreamState, SharedStreamCtx, SharedStreamManager,
StreamAdmissionError, StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest,
UserSession,
},
},
auth::Fingerprint,
@@ -189,7 +190,11 @@ pub(crate) fn admission_failure_response(
/// players receive an unambiguous error instead of an empty success body.
fn stream_admission_rejected_response(error: StreamAdmissionError, username: &str) -> axum::response::Response {
error!("Stream admission rejected for user {username}: {error:?}");
StatusCode::SERVICE_UNAVAILABLE.into_response()
axum::response::Response::builder()
.status(StatusCode::SERVICE_UNAVAILABLE)
.header("x-tuliprox-rejection", "admission_rejected")
.body(axum::body::Body::empty())
.unwrap_or_else(|_| StatusCode::SERVICE_UNAVAILABLE.into_response())
}
#[macro_export]
@@ -1156,7 +1161,7 @@ fn create_unmapped_provider_stream(app_config: &AppConfig) -> ProviderStreamStat
}
}
fn acquire_stream_provider_handle(
async fn acquire_stream_provider_handle(
app_state: &Arc<AppState>,
input: &ConfigInput,
fingerprint: &Fingerprint,
@@ -1167,14 +1172,18 @@ fn acquire_stream_provider_handle(
match options.force_provider {
Some(provider) => {
// First try to stay on the exact pinned provider account without over-allocating.
if let Some(handle) = app_state.active_provider.acquire_exact_connection_with_lease_for_session(
provider,
&fingerprint.addr,
options.allow_provider_grace,
options.user_priority,
options.connection_kind,
lease,
) {
if let Some(handle) = app_state
.active_provider
.acquire_exact_connection_with_lease_for_session_await(
provider,
&fingerprint.addr,
options.allow_provider_grace,
options.user_priority,
options.connection_kind,
lease,
)
.await
{
Some(managed(handle))
} else if options.allow_forced_provider_fallback {
debug_if_enabled!(
@@ -1184,7 +1193,7 @@ fn acquire_stream_provider_handle(
);
app_state
.active_provider
.acquire_connection_with_lease_for_session(
.acquire_connection_with_lease_for_session_await(
&input.name,
&fingerprint.addr,
options.allow_provider_grace,
@@ -1192,6 +1201,7 @@ fn acquire_stream_provider_handle(
options.connection_kind,
lease,
)
.await
.map(managed)
} else {
debug_if_enabled!(
@@ -1204,7 +1214,7 @@ fn acquire_stream_provider_handle(
}
None => app_state
.active_provider
.acquire_connection_with_lease_for_session(
.acquire_connection_with_lease_for_session_await(
&input.name,
&fingerprint.addr,
options.allow_provider_grace,
@@ -1212,6 +1222,7 @@ fn acquire_stream_provider_handle(
options.connection_kind,
lease,
)
.await
.map(managed),
}
}
@@ -1269,7 +1280,7 @@ async fn resolve_streaming_strategy(
) -> StreamingStrategy {
// allocate a provider connection
let accept_requested_stream_url = options.accept_requested_stream_url || input.input_type.is_stalker();
let mut provider_connection_handle = acquire_stream_provider_handle(app_state, input, fingerprint, &options);
let mut provider_connection_handle = acquire_stream_provider_handle(app_state, input, fingerprint, &options).await;
// panel_api provisioning/loading is handled later in the stream creation flow
@@ -1556,7 +1567,7 @@ async fn create_stream_response_details(
match streaming_strategy.provider_stream_state {
// custom stream means we display our own stream like connection exhausted, channel-unavailable...
ProviderStreamState::Custom { response: provider_stream, .. } => {
ProviderStreamState::Custom { response: provider_stream, reason } => {
let (stream, stream_info) = provider_stream;
// When allocation is exhausted or no connection was acquired, guard_provider_name is None.
// Use input.name as fallback so the provider field is never empty.
@@ -1577,6 +1588,7 @@ async fn create_stream_response_details(
provider_handle: streaming_strategy.provider_handle.take(),
content_representation,
grace_resolution_context,
custom_reason: Some(reason),
})
}
ProviderStreamState::Available(_provider_name, request_url)
@@ -1654,10 +1666,15 @@ async fn create_stream_response_details(
}
let reconnect_flag = provider_stream_factory_options.get_reconnect_flag_clone();
let provider_stream = match create_provider_stream(
let lifecycle = streaming_strategy
.provider_handle
.as_ref()
.and_then(ProviderStreamOpenLifecycle::from_managed);
let provider_stream = match open_provider_stream_with_lifecycle(
&app_state.provider_stream_ctx(),
&app_state.http_client.load(),
provider_stream_factory_options,
lifecycle,
)
.await
{
@@ -1717,11 +1734,19 @@ async fn create_stream_response_details(
}
options.set_provider(input.get_resolve_provider(url.as_ref()));
options.require_public_destination();
if let Some(m) = streaming_strategy.provider_handle.as_mut() {
let _ = m.renew_opening_tokens();
}
let retry_reconnect_flag = options.get_reconnect_flag_clone();
let retried = create_provider_stream(
let retry_lifecycle = streaming_strategy
.provider_handle
.as_ref()
.and_then(ProviderStreamOpenLifecycle::from_managed);
let retried = open_provider_stream_with_lifecycle(
&app_state.provider_stream_ctx(),
&app_state.http_client.load(),
options,
retry_lifecycle,
)
.await;
if let Some(response) = retried {
@@ -1790,6 +1815,7 @@ async fn create_stream_response_details(
provider_handle,
content_representation,
grace_resolution_context,
custom_reason: None,
})
}
}
@@ -2086,7 +2112,7 @@ pub async fn force_provider_stream_response(
sanitize_sensitive_info(&user_session.token),
sanitize_sensitive_info(&fingerprint.addr.to_string())
);
cleanup_forced_reopen_addrs(app_state, &user_session.token, &cleanup_addrs);
cleanup_forced_reopen_addrs(app_state, &user_session.token, &cleanup_addrs).await;
}
// Provider-affine playback must stay on the same provider account across seeks/range reconnects.
@@ -2742,6 +2768,24 @@ pub(crate) async fn stream_response(
activation.placeholder_transition_version.is_some(),
)
.await;
if stream_details.custom_reason == Some(ProviderStreamCustomReason::ProviderExhausted) {
record_connect_failed_attempt(ConnectFailedAttempt {
app_state,
fingerprint,
user,
stream_channel,
provider_name: stream_details.provider_name.unwrap_or_else(|| input.name.clone()),
req_headers,
reason: ConnectFailureReason::ProviderConnectionsExhausted,
failure_stage: FailureStage::Admission,
});
return create_custom_video_stream_response(
&app_state.provider_stream_ctx(),
&fingerprint.addr,
CustomVideoStreamType::ProviderConnectionsExhausted,
)
.into_response();
}
no_custom_video_fallback_status(&app_state.app_config).into_response()
}
@@ -2991,8 +3035,8 @@ pub(crate) fn get_hls_session_ttl_secs(app_state: &Arc<AppState>) -> u64 {
get_stream_config_u64(app_state, |stream| stream.hls_session_ttl_secs, default_hls_session_ttl_secs())
}
fn cleanup_forced_reopen_addrs(app_state: &Arc<AppState>, session_owner: &str, cleanup_addrs: &[SocketAddr]) {
app_state.active_provider.release_playback_connections(session_owner, cleanup_addrs);
async fn cleanup_forced_reopen_addrs(app_state: &Arc<AppState>, session_owner: &str, cleanup_addrs: &[SocketAddr]) {
app_state.active_provider.release_playback_connections_await(session_owner, cleanup_addrs).await;
}
pub(crate) fn get_catchup_session_ttl_secs(app_state: &Arc<AppState>) -> u64 {
backend/app/src/api/api_utils/tests.rs
+147 -2
View File
@@ -7200,6 +7200,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid
provider_handle: None,
content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: None,
custom_reason: None,
};
assert!(
should_pin_provider_for_session(&no_video_details, &app_state, PlaylistItemType::Catchup),
@@ -7222,6 +7223,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid
provider_handle: None,
content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: None,
custom_reason: None,
};
assert!(
should_pin_provider_for_session(&provisioning_details, &app_state, PlaylistItemType::Catchup),
@@ -7251,6 +7253,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid
provider_handle: None,
content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: None,
custom_reason: None,
};
assert!(
!should_pin_provider_for_session(&failure_details, &app_state, PlaylistItemType::Catchup),
@@ -8276,7 +8279,7 @@ async fn forced_reopen_cleanup_for_adaptive_streams_does_not_close_client_socket
let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!());
let mut close_rx = app_state.connection_manager.get_close_connection_channel();
cleanup_forced_reopen_addrs(&app_state, "adaptive-owner", &[addr]);
cleanup_forced_reopen_addrs(&app_state, "adaptive-owner", &[addr]).await;
let signal =
tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok);
@@ -8289,7 +8292,7 @@ async fn forced_reopen_cleanup_for_non_adaptive_streams_preserves_client_socket(
let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!());
let mut close_rx = app_state.connection_manager.get_close_connection_channel();
cleanup_forced_reopen_addrs(&app_state, "live-owner", &[addr]);
cleanup_forced_reopen_addrs(&app_state, "live-owner", &[addr]).await;
let signal =
tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok);
@@ -8693,3 +8696,145 @@ fn evaluate_network_access_respects_allow_policy() {
NetworkAccessDecision::AllowedGeoIpUnavailable
);
}
#[tokio::test]
async fn test_channel_unavailable_fallback_does_not_register_body_owner_or_leak_slot() {
let app_state = create_test_provider_app_state();
let provider_name = "provider_1".intern();
let mut cfg = (**app_state.app_config.config.load()).clone();
cfg.custom_stream_response_enabled = true;
app_state.app_config.config.store(std::sync::Arc::new(cfg));
let addr: std::net::SocketAddr = "127.0.0.1:49080".parse().unwrap();
let custom_video = crate::model::CustomStreamResponse {
channel_unavailable: Some(crate::api::model::TransportStreamBuffer::new(b"channel-unavailable-bytes".to_vec())),
user_connections_exhausted: None,
provider_connections_exhausted: None,
low_priority_preempted: None,
user_account_expired: None,
panel_api_provisioning: None,
hls_session_or_lease_expired: None,
panel_api_provisioning_hls_segments: Vec::new(),
};
app_state.app_config.custom_stream_response.store(Some(std::sync::Arc::new(custom_video)));
let handle = app_state
.active_provider
.acquire_connection(&provider_name, &addr, 0, tuliprox_session::ConnectionKind::Normal)
.expect("connection should be acquired");
assert_eq!(app_state.active_provider.get_provider_connections_count(), 1);
let (channel_unavail_stream, info) =
create_channel_unavailable_stream(&app_state.app_config, &[], axum::http::StatusCode::OK);
assert!(channel_unavail_stream.is_some());
let factory_response = tuliprox_session::ProviderStreamFactoryResponse {
stream: channel_unavail_stream.unwrap(),
info,
provider_session_headers: std::collections::HashMap::new(),
has_upstream_owner: false,
};
if factory_response.has_upstream_owner {
app_state.active_provider.register_body_owner(handle.allocation_id);
}
app_state.active_provider.release_handle(&handle);
assert_eq!(app_state.active_provider.get_provider_connections_count(), 0);
assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::<usize>()).unwrap_or(0), 0);
}
#[tokio::test]
async fn test_provider_open_cancelled_during_header_wait_releases_slot_without_leak() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let local_addr = listener.local_addr().unwrap();
let accepted = std::sync::Arc::new(tokio::sync::Notify::new());
let accepted_clone = std::sync::Arc::clone(&accepted);
let server_task = tokio::spawn(async move {
if let Ok((mut socket, _)) = listener.accept().await {
accepted_clone.notify_waiters();
let mut buf = [0u8; 1024];
loop {
use tokio::io::AsyncReadExt;
match socket.read(&mut buf).await {
Ok(0) | Err(_) => break,
Ok(_) => {}
}
}
}
});
let app_state = create_test_provider_app_state();
let provider_name = "provider_1".intern();
let client_addr: std::net::SocketAddr = "127.0.0.1:49081".parse().unwrap();
let handle = app_state
.active_provider
.acquire_connection(&provider_name, &client_addr, 0, tuliprox_session::ConnectionKind::Normal)
.expect("connection should be acquired");
assert_eq!(app_state.active_provider.get_provider_connections_count(), 1);
let url = url::Url::parse(&format!("http://{local_addr}/stream.ts")).unwrap();
let req_headers = axum::http::HeaderMap::new();
let stream_options = get_stream_options(&app_state.app_config);
let mut options =
crate::api::model::ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams {
addr: client_addr,
item_type: PlaylistItemType::Live,
share_stream: false,
stream_options: &stream_options,
stream_url: &url,
req_headers: &req_headers,
input_headers: None,
session_headers: None,
disabled_headers: None,
default_user_agent: None,
username: None,
client_ip: None,
stream_channel: None,
connect_failure_stage: None,
content_representation: tuliprox_session::ProviderContentRepresentationMode::PreserveOrigin,
});
options.set_provider_handle_tokens(
handle.cancel_token.clone(),
handle.completion_token.clone(),
Some(handle.close_reason.clone()),
);
app_state.active_provider.mark_opening(handle.allocation_id);
let ctx = app_state.provider_stream_ctx();
let client = app_state.http_client.load().as_ref().clone();
let open_task =
tokio::spawn(async move { crate::api::model::create_provider_stream(&ctx, &client, options).await });
// Wait until TCP connection is accepted and waiting for response headers
accepted.notified().await;
// Now cancel the handle while it is waiting for response headers
handle.cancel_token.as_ref().unwrap().cancel();
// The open task should finish quickly because it is cancelled
let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), open_task)
.await
.expect("open task should abort upon cancellation")
.expect("task join");
assert!(outcome.is_none());
// Release the handle
app_state.active_provider.release_handle(&handle);
// Yield to allow background completion tasks to process
tokio::task::yield_now().await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(app_state.active_provider.get_provider_connections_count(), 0);
assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::<usize>()).unwrap_or(0), 0);
let _ = server_task.await;
}
backend/app/src/api/model/streams/active_client_stream.rs
+20 -6
View File
@@ -2,10 +2,10 @@ use crate::{
api::{
model::{
connection_manager::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET},
create_provider_stream, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, CleanupEvent,
ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream, PendingProviderWakeSource,
ProviderStreamFactoryOptions, StreamDetails, StreamError, StreamMeterHandle, TimedClientStream,
TransportStreamBuffer,
open_provider_stream_with_lifecycle, uses_direct_body_idle_timeout, AppState, BoxedProviderStream,
CleanupEvent, ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream,
PendingProviderWakeSource, ProviderStreamFactoryOptions, ProviderStreamOpenLifecycle, StreamDetails,
StreamError, StreamMeterHandle, TimedClientStream, TransportStreamBuffer,
},
panel_api::{can_provision_on_exhausted, find_input_by_provider_name, run_panel_api_provisioning_probe},
},
@@ -421,6 +421,12 @@ impl ActiveClientStreamState {
self.preempt_cancelled = None;
self.stop_grace_task();
let is_superseded = self
.provider_handle
.as_ref()
.and_then(|m| m.handle())
.is_some_and(|h| h.get_close_reason() == tuliprox_core::model::ProviderCloseReason::Superseded);
let mut serve_preempted_custom = false;
if self.provider_handle.is_some() {
let managed = self.provider_handle.take();
@@ -428,7 +434,7 @@ impl ActiveClientStreamState {
// Synchronous provider-slot release; the custom-video detail update is a
// separate, best-effort UI effect.
drop(managed);
if self.custom_video.low_priority_preempted.is_some() {
if !is_superseded && self.custom_video.low_priority_preempted.is_some() {
serve_preempted_custom = true;
if let Some(flag) = &self.send_custom_stream_flag {
flag.store(StreamMode::LowPriorityPreempted as u8, Ordering::Release);
@@ -706,11 +712,17 @@ impl Stream for ActiveClientStream {
let http_client = app_state.http_client.load();
http_client.as_ref().clone()
};
let lifecycle = self
.state
.provider_handle
.as_ref()
.and_then(ProviderStreamOpenLifecycle::from_managed);
let future = Box::pin(async move {
match create_provider_stream(
match open_provider_stream_with_lifecycle(
&app_state.provider_stream_ctx(),
&client,
context.provider_stream_factory_options,
lifecycle,
)
.await
{
@@ -1995,6 +2007,7 @@ mod tests {
provider_handle: Some(provider_handle),
content_representation: ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: None,
custom_reason: None,
}
}
@@ -2934,6 +2947,7 @@ mod tests {
provider_handle: None,
content_representation: ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: Some(grace_context.clone()),
custom_reason: None,
};
let (flag, grace_task) = stream_grace_period(GracePeriodParams {
backend/app/src/api/model/streams/mod.rs
+1
View File
@@ -45,4 +45,5 @@ pub use crate::utils::network::request::STREAM_IDLE_TIMEOUT;
pub(crate) use shared::model::CustomVideoStreamType;
// Shared-stream fan-out and its buffer moved to `tuliprox-session`;
// re-exported so api call sites keep their names, module paths included.
#[cfg(test)]
pub(crate) use tuliprox_session::streams::buffered_stream;
backend/app/src/api/model/streams/provider_stream.rs
+12 -1
View File
@@ -230,7 +230,18 @@ pub fn create_custom_video_stream_response(
// configured `custom_stream_response_error_status` (default 502) instead of a
// hard-coded 403, so a reverse proxy with `proxy_intercept_errors on;` can sever
// the socket.
get_custom_stream_response_error_status(config).into_response()
let status = get_custom_stream_response_error_status(config);
let mut builder = axum::response::Response::builder().status(status);
if matches!(
video_response,
CustomVideoStreamType::UserConnectionsExhausted
| CustomVideoStreamType::ProviderConnectionsExhausted
| CustomVideoStreamType::LowPriorityPreempted
| CustomVideoStreamType::UserAccountExpired
) {
builder = builder.header("x-tuliprox-rejection", "admission_rejected");
}
builder.body(axum::body::Body::empty()).unwrap_or_else(|_| status.into_response())
}
pub fn get_header_filter_for_item_type(item_type: PlaylistItemType) -> HeaderFilter {
match item_type {
backend/app/src/api/model/streams/provider_stream_factory.rs
+192 -19
View File
@@ -1,9 +1,8 @@
use crate::{
api::model::{
create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers,
streams::{buffered_stream::BufferedStream, client_stream::ClientStream},
CustomVideoStreamType, ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError,
STREAM_IDLE_TIMEOUT,
streams::client_stream::ClientStream, CustomVideoStreamType, ProviderContentRepresentationMode,
ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT,
},
iptv::stalker::client::validate_public_playable_url,
model::{AppConfig, ConfigProvider, ReverseProxyDisabledHeaderConfig},
@@ -36,11 +35,11 @@ use std::{
error::Error as StdError,
io,
net::SocketAddr,
sync::Arc,
sync::{atomic::AtomicU8, Arc},
time::{Duration, Instant},
};
use tokio_util::{io::ReaderStream, sync::CancellationToken};
use tuliprox_core::utils::request_headers::get_headers_from_request;
use tuliprox_core::{model::AllocationId, utils::request_headers::get_headers_from_request};
use tuliprox_hls::api::{
extract_hls_provider_session_headers, log_hls_origin_content_coding, HlsOriginContentCodingObjectKind,
HlsOriginContentCodingSource,
@@ -49,6 +48,8 @@ use tuliprox_session::{
response_headers::{provider_response_headers, ProviderResponseHeaderError},
stream_ctx::ProviderStreamCtx,
stream_options::StreamOptions,
streams::{ProviderBodyOwner, ProviderBodyOwnerConfig},
ActiveProviderManager, ManagedProviderHandle,
};
use url::Url;
@@ -103,6 +104,9 @@ pub struct ProviderStreamFactoryOptions {
content_representation: ProviderContentRepresentationMode,
response_head_availability: ProviderResponseHeadAvailability,
hls_content_coding_object_kind: Option<HlsOriginContentCodingObjectKind>,
cancel_token: Option<CancellationToken>,
completion_token: Option<CancellationToken>,
close_reason: Option<Arc<AtomicU8>>,
}
pub(crate) struct ProviderStreamFactoryParams<'a> {
@@ -217,9 +221,29 @@ impl ProviderStreamFactoryOptions {
ProviderContentRepresentationMode::Identity => Some(HlsOriginContentCodingObjectKind::Other),
ProviderContentRepresentationMode::PreserveOrigin => None,
},
cancel_token: None,
completion_token: None,
close_reason: None,
}
}
pub fn set_provider_handle_tokens(
&mut self,
cancel_token: Option<CancellationToken>,
completion_token: Option<CancellationToken>,
close_reason: Option<Arc<AtomicU8>>,
) {
self.cancel_token = cancel_token;
self.completion_token = completion_token;
self.close_reason = close_reason;
}
pub fn get_cancel_token(&self) -> Option<CancellationToken> { self.cancel_token.clone() }
pub fn get_completion_token(&self) -> Option<CancellationToken> { self.completion_token.clone() }
pub fn get_close_reason(&self) -> Option<Arc<AtomicU8>> { self.close_reason.clone() }
pub fn set_provider(&mut self, provider: Option<Arc<ConfigProvider>>) { self.provider = provider; }
pub fn apply_user_agent_stream_index(&mut self, stream_index: u64) {
@@ -748,7 +772,12 @@ async fn prepare_provider_stream_response_with_context(
let headers = provider_response_headers(response.headers(), context.representation)?;
let response_info = Some((headers, response.status(), Some(response.url().clone()), None));
let stream = response.bytes_stream().map_err(|error| StreamError::reqwest(&error)).boxed();
Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers })
Ok(ProviderStreamFactoryResponse {
stream,
info: response_info,
provider_session_headers,
has_upstream_owner: false,
})
}
ProviderContentRepresentationMode::Identity => {
let origin_status = response.status();
@@ -787,7 +816,12 @@ async fn prepare_provider_stream_response_with_context(
let headers = provider_response_headers(&decoded.headers, context.representation)?;
let response_info = Some((headers, decoded.status, Some(decoded.final_url), None));
let stream = ReaderStream::new(decoded.body).map_err(|error| provider_decoded_body_error(&error)).boxed();
Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers })
Ok(ProviderStreamFactoryResponse {
stream,
info: response_info,
provider_session_headers,
has_upstream_owner: false,
})
}
}
}
@@ -963,6 +997,7 @@ async fn provider_stream_request(
}
}
#[allow(clippy::too_many_lines)]
async fn get_provider_stream(
ctx: &ProviderStreamCtx,
client: &reqwest::Client,
@@ -973,8 +1008,29 @@ async fn get_provider_stream(
let start = Instant::now();
let mut connect_err: u32 = 1;
let handle_cancel = stream_options.get_cancel_token();
while stream_options.should_continue() {
match provider_stream_request(ctx, client, stream_options).await {
let request_future = provider_stream_request(ctx, client, stream_options);
let request_res = if let Some(ref cancel) = handle_cancel {
tokio::select! {
biased;
() = cancel.cancelled() => {
debug!(
"Provider stream request cancelled during open: {}",
sanitize_sensitive_info(stream_options.get_log_url().as_ref())
);
return Err(ProviderStreamRequestFailure::Status {
status: StatusCode::BAD_GATEWAY,
provider_error_class: "cancelled",
serve_channel_unavailable: false,
});
}
res = request_future => res,
}
} else {
request_future.await
};
match request_res {
Ok(Some(stream_response)) => {
return Ok(Some(stream_response));
}
@@ -1030,7 +1086,15 @@ async fn get_provider_stream(
break;
}
connect_err += 1;
tokio::time::sleep(Duration::from_millis(50)).await;
if let Some(ref cancel) = handle_cancel {
tokio::select! {
biased;
() = cancel.cancelled() => break,
() = tokio::time::sleep(Duration::from_millis(50)) => {},
}
} else {
tokio::time::sleep(Duration::from_millis(50)).await;
}
debug_if_enabled!("Reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref()));
}
debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref()));
@@ -1042,14 +1106,31 @@ async fn get_provider_stream(
})
}
struct ProviderOpenGuard {
token: Option<CancellationToken>,
handed_off: bool,
}
impl Drop for ProviderOpenGuard {
fn drop(&mut self) {
if !self.handed_off {
if let Some(token) = self.token.take() {
token.cancel();
}
}
}
}
#[allow(clippy::too_many_lines)]
pub async fn create_provider_stream(
ctx: &ProviderStreamCtx,
client: &reqwest::Client,
stream_options: ProviderStreamFactoryOptions,
) -> Option<ProviderStreamFactoryResponse> {
let mut open_guard = ProviderOpenGuard { token: stream_options.get_completion_token(), handed_off: false };
match get_provider_stream(ctx, client, &stream_options).await {
Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers })) => {
Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers, .. })) => {
let continue_signal = stream_options.get_reconnect_flag_clone();
if let Some((_headers, _status, _response_url, Some(custom_video_type))) = &info {
let reason = match custom_video_type {
CustomVideoStreamType::ChannelUnavailable => Some(ConnectFailureReason::ChannelUnavailable),
@@ -1062,26 +1143,66 @@ pub async fn create_provider_stream(
if let Some(reason) = reason {
record_provider_open_failure(ctx, &stream_options, reason, None, None);
}
return Some(ProviderStreamFactoryResponse {
stream: ClientStream::new(init_stream, continue_signal, None, stream_options.get_url_as_str())
.boxed(),
info,
provider_session_headers,
has_upstream_owner: false,
});
}
let continue_signal = stream_options.get_reconnect_flag_clone();
let stream = init_stream.boxed();
let stream = if should_wrap_provider_stream_in_buffer(&stream_options) {
BufferedStream::new(
stream,
let handle_cancel = stream_options.get_cancel_token();
let completion_token = stream_options.get_completion_token();
let close_reason = stream_options.get_close_reason();
let cancel_token = if let Some(hc) = handle_cancel {
let combined = CancellationToken::new();
let c1 = combined.clone();
let c2 = combined.clone();
let hc_clone = hc.clone();
let cs_clone = continue_signal.clone();
tokio::spawn(async move {
tokio::select! {
biased;
() = hc_clone.cancelled() => {
cs_clone.cancel();
c1.cancel();
}
() = cs_clone.cancelled() => {
c2.cancel();
}
}
});
combined
} else {
continue_signal.clone()
};
let body_owner_config = if should_wrap_provider_stream_in_buffer(&stream_options) {
ProviderBodyOwnerConfig::for_buffered(
stream_options.get_buffer_size(),
stream_options.get_buffer_max_bytes(),
stream_options.get_reconnect_flag_clone(),
stream_options.get_url_as_str(),
)
.boxed()
} else {
stream
ProviderBodyOwnerConfig::for_direct_body()
};
open_guard.handed_off = true;
let stream = ProviderBodyOwner::new(
init_stream.boxed(),
body_owner_config,
cancel_token,
completion_token,
close_reason,
)
.boxed();
Some(ProviderStreamFactoryResponse {
stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str())
.boxed(),
info,
provider_session_headers,
has_upstream_owner: true,
})
}
Ok(None) => None,
@@ -1103,6 +1224,7 @@ pub async fn create_provider_stream(
stream: boxed_provider_stream,
info: response_info,
provider_session_headers: HashMap::new(),
has_upstream_owner: false,
});
}
None
@@ -1110,6 +1232,57 @@ pub async fn create_provider_stream(
}
}
#[derive(Clone)]
pub struct ProviderStreamOpenLifecycle {
manager: Arc<ActiveProviderManager>,
allocation_id: AllocationId,
cancel_token: Option<CancellationToken>,
completion_token: Option<CancellationToken>,
close_reason: Option<Arc<AtomicU8>>,
}
impl ProviderStreamOpenLifecycle {
pub fn from_managed(managed: &ManagedProviderHandle) -> Option<Self> {
let handle = managed.handle()?;
Some(Self {
manager: Arc::clone(managed.manager()),
allocation_id: handle.allocation_id,
cancel_token: handle.cancel_token.clone(),
completion_token: handle.completion_token.clone(),
close_reason: Some(Arc::clone(&handle.close_reason)),
})
}
pub fn mark_opening(&self) { self.manager.mark_opening(self.allocation_id); }
pub fn register_body_owner(&self) -> bool { self.manager.register_body_owner(self.allocation_id) }
}
pub async fn open_provider_stream_with_lifecycle(
ctx: &ProviderStreamCtx,
client: &reqwest::Client,
mut stream_options: ProviderStreamFactoryOptions,
lifecycle: Option<ProviderStreamOpenLifecycle>,
) -> Option<ProviderStreamFactoryResponse> {
if let Some(ref lc) = lifecycle {
lc.mark_opening();
stream_options.set_provider_handle_tokens(
lc.cancel_token.clone(),
lc.completion_token.clone(),
lc.close_reason.clone(),
);
}
let response = create_provider_stream(ctx, client, stream_options).await?;
if response.has_upstream_owner {
if let Some(ref lc) = lifecycle {
if !lc.register_body_owner() {
return None;
}
}
}
Some(response)
}
#[cfg(test)]
mod tests {
use super::*;
backend/app/src/api/panel_api/mod.rs
+2
View File
@@ -3692,6 +3692,7 @@ pub fn create_panel_api_provisioning_stream_details(
provider_handle: None,
content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: None,
custom_reason: None,
};
}
@@ -3722,6 +3723,7 @@ pub fn create_panel_api_provisioning_stream_details(
provider_handle: None,
content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: None,
custom_reason: None,
}
}
backend/core/src/model/provider.rs
+61 -1
View File
@@ -94,6 +94,42 @@ impl ProviderAllocation {
}
}
use std::sync::atomic::{AtomicU8, Ordering};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConnectionLifecycle {
Opening,
Active,
Closing,
Closed,
}
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProviderCloseReason {
Unspecified = 0,
Superseded = 1,
PriorityPreempted = 2,
ClientClosed = 3,
IdleTimeout = 4,
ProviderError = 5,
Shutdown = 6,
}
impl ProviderCloseReason {
pub fn from_u8(v: u8) -> Self {
match v {
1 => Self::Superseded,
2 => Self::PriorityPreempted,
3 => Self::ClientClosed,
4 => Self::IdleTimeout,
5 => Self::ProviderError,
6 => Self::Shutdown,
_ => Self::Unspecified,
}
}
}
#[derive(Debug, Clone)]
pub struct ProviderHandle {
pub playback_request_id: Option<super::PlaybackRequestId>,
@@ -106,6 +142,8 @@ pub struct ProviderHandle {
pub allocation: ProviderAllocation,
// Token to cancel the background task (e.g. internal probe) if preempted
pub cancel_token: Option<CancellationToken>,
pub completion_token: Option<CancellationToken>,
pub close_reason: Arc<AtomicU8>,
}
impl ProviderHandle {
@@ -115,7 +153,29 @@ impl ProviderHandle {
allocation: ProviderAllocation,
cancel_token: Option<CancellationToken>,
) -> Self {
Self { client_id, allocation_id, allocation, cancel_token, playback_request_id: None, binding_tag: None }
Self {
client_id,
allocation_id,
allocation,
cancel_token,
completion_token: None,
close_reason: Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8)),
playback_request_id: None,
binding_tag: None,
}
}
pub fn set_close_reason(&self, reason: ProviderCloseReason) {
let _ = self.close_reason.compare_exchange(
ProviderCloseReason::Unspecified as u8,
reason as u8,
Ordering::AcqRel,
Ordering::Relaxed,
);
}
pub fn get_close_reason(&self) -> ProviderCloseReason {
ProviderCloseReason::from_u8(self.close_reason.load(Ordering::Acquire))
}
}
backend/parser/src/m3u.rs
+2
View File
@@ -187,6 +187,8 @@ fn classify_token(t: &str) -> M3uToken {
4 => {
if eq_ascii(bytes, b"cuid") {
M3uToken::ProviderId
} else if eq_ascii(bytes, b"type") {
M3uToken::TvgType
} else {
M3uToken::Unknown
}
backend/session/src/active_provider_manager.rs
+1025 -302
View File
File diff suppressed because it is too large Load Diff
backend/session/src/active_user_manager/mod.rs
+1 -1
View File
@@ -1556,7 +1556,7 @@ impl ActiveUserManager {
addr_count == 1
}
})
.map(|s| crate::EvictionCandidate { addr: s.addr, client_ip: s.client_ip.clone(), ts: s.ts })
.map(|s| crate::EvictionCandidate { addr: s.addr, client_ip: s.client_ip.clone(), ts: s.ts, uid: s.uid })
.collect();
candidates
}
backend/session/src/admission_strategy.rs
+12 -3
View File
@@ -109,8 +109,8 @@ fn select_candidate<'a>(
let should_replace = match selected {
None => true,
Some(current) => match order {
EvictionOrder::Oldest => candidate.ts < current.ts,
EvictionOrder::Latest => candidate.ts > current.ts,
EvictionOrder::Oldest => (candidate.ts, candidate.uid) < (current.ts, current.uid),
EvictionOrder::Latest => (candidate.ts, candidate.uid) > (current.ts, current.uid),
},
};
if should_replace {
@@ -125,6 +125,8 @@ pub struct EvictionCandidate {
pub addr: SocketAddr,
pub client_ip: String,
pub ts: u64,
/// Monotonic stream identity used to order streams created in the same second.
pub uid: u32,
}
#[cfg(test)]
@@ -135,7 +137,7 @@ mod tests {
fn addr(port: u16) -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port) }
fn candidate(port: u16, ip: &str, ts: u64) -> EvictionCandidate {
EvictionCandidate { addr: addr(port), client_ip: ip.to_string(), ts }
EvictionCandidate { addr: addr(port), client_ip: ip.to_string(), ts, uid: u32::from(port) }
}
#[test]
@@ -183,6 +185,13 @@ mod tests {
assert_evict(AdmissionStrategy::EvictUserLatest, &candidates, 1000);
}
#[test]
fn equal_timestamp_uses_monotonic_stream_identity() {
let candidates = vec![candidate(1000, "1.1.1.1", 100), candidate(1001, "1.1.1.1", 100)];
assert_evict(AdmissionStrategy::EvictUserSameIpOldest, &candidates, 1000);
assert_evict(AdmissionStrategy::EvictUserSameIpLatest, &candidates, 1001);
}
#[test]
fn no_same_ip_candidate_returns_no_match() {
let ctx = StrategyContext { username: "user1", client_ip: "1.1.1.1" };
backend/session/src/stream.rs
+4
View File
@@ -21,6 +21,7 @@ pub struct ProviderStreamFactoryResponse {
pub stream: BoxedProviderStream,
pub info: ProviderStreamInfo,
pub provider_session_headers: HashMap<String, String>,
pub has_upstream_owner: bool,
}
/// Controls whether a provider stream preserves its origin representation or normalizes it to identity bytes.
@@ -85,6 +86,7 @@ pub struct StreamDetails {
/// Set when the stream was admitted via a user-grace strategy. Carried through to
/// `stream_grace_period` so remaining strategies can be evaluated if the grace fails.
pub grace_resolution_context: Option<crate::GraceResolutionContext>,
pub custom_reason: Option<ProviderStreamCustomReason>,
}
/// Manual Clone: stream and `provider_handle` cannot be duplicated so we set them to None on the clone.
@@ -107,6 +109,7 @@ impl Clone for StreamDetails {
provider_handle: None,
content_representation: self.content_representation,
grace_resolution_context: self.grace_resolution_context.clone(),
custom_reason: self.custom_reason,
}
}
}
@@ -129,6 +132,7 @@ impl StreamDetails {
provider_handle: None,
content_representation: ProviderContentRepresentationMode::PreserveOrigin,
grace_resolution_context: None,
custom_reason: None,
}
}
#[inline]
backend/session/src/streams/mod.rs
+2 -1
View File
@@ -1,4 +1,5 @@
pub mod buffered_stream;
pub mod provider_body_owner;
pub mod shared_stream_manager;
pub use self::{buffered_stream::*, shared_stream_manager::*};
pub use self::{buffered_stream::*, provider_body_owner::*, shared_stream_manager::*};
backend/session/src/streams/provider_body_owner.rs
+375
View File
@@ -0,0 +1,375 @@
use crate::BoxedProviderStream;
use bytes::Bytes;
use futures::{
stream::Stream,
task::{Context, Poll},
StreamExt,
};
use log::debug;
use std::{
cmp::max,
future::Future,
pin::Pin,
sync::{
atomic::{AtomicU8, Ordering},
Arc,
},
time::Duration,
};
use tokio::{
select,
sync::{
mpsc::{channel, error::TrySendError, Sender},
Semaphore,
},
time::{sleep, Instant},
};
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned};
use tuliprox_core::model::{ProviderCloseReason, StreamError};
pub const DEFAULT_DIRECT_BODY_CHANNEL_SIZE: usize = 2;
pub const DEFAULT_DIRECT_BODY_MAX_BYTES: usize = 256 * 1024;
pub const DEFAULT_DIRECT_BODY_IDLE_SECS: u64 = 90;
pub const DEFAULT_BUFFERED_CHANNEL_SIZE: usize = 1024;
pub const DEFAULT_BUFFERED_MAX_BYTES: usize = 5 * 1024 * 1024;
pub const DEFAULT_BUFFERED_IDLE_SECS: u64 = 10;
#[derive(Debug, Clone, Copy)]
pub struct ProviderBodyOwnerConfig {
pub channel_capacity: usize,
pub max_buffer_bytes: usize,
pub idle_timeout: Duration,
}
impl ProviderBodyOwnerConfig {
pub fn for_direct_body() -> Self {
Self {
channel_capacity: DEFAULT_DIRECT_BODY_CHANNEL_SIZE,
max_buffer_bytes: DEFAULT_DIRECT_BODY_MAX_BYTES,
idle_timeout: Duration::from_secs(DEFAULT_DIRECT_BODY_IDLE_SECS),
}
}
pub fn for_buffered(buffer_size: usize, max_buffer_bytes: usize) -> Self {
Self {
channel_capacity: max(buffer_size, DEFAULT_BUFFERED_CHANNEL_SIZE),
max_buffer_bytes: if max_buffer_bytes == 0 { DEFAULT_BUFFERED_MAX_BYTES } else { max_buffer_bytes },
idle_timeout: Duration::from_secs(DEFAULT_BUFFERED_IDLE_SECS),
}
}
}
/// An upstream body owner task that decouples upstream body consumption from downstream polling.
/// It observes cancellation independently of downstream `poll_next`, ensuring that upstream
/// connections close promptly upon preemption or client disconnect.
pub struct ProviderBodyOwner {
stream: ReceiverStream<Result<Bytes, StreamError>>,
close_cancelled: Pin<Box<WaitForCancellationFutureOwned>>,
cancel_token: CancellationToken,
semaphore: Arc<Semaphore>,
max_buffer_bytes: usize,
}
impl ProviderBodyOwner {
pub fn new(
stream: BoxedProviderStream,
config: ProviderBodyOwnerConfig,
cancel_token: CancellationToken,
completion_token: Option<CancellationToken>,
close_reason: Option<Arc<AtomicU8>>,
) -> Self {
let (tx, rx) = channel(config.channel_capacity);
let semaphore = Arc::new(Semaphore::new(config.max_buffer_bytes));
tokio::spawn(Self::run_owner_task(
tx,
stream,
cancel_token.clone(),
completion_token,
close_reason,
Arc::clone(&semaphore),
config,
));
Self {
stream: ReceiverStream::new(rx),
close_cancelled: Box::pin(cancel_token.clone().cancelled_owned()),
cancel_token,
semaphore,
max_buffer_bytes: config.max_buffer_bytes,
}
}
async fn run_owner_task(
tx: Sender<Result<Bytes, StreamError>>,
mut stream: BoxedProviderStream,
cancel_token: CancellationToken,
completion_token: Option<CancellationToken>,
close_reason: Option<Arc<AtomicU8>>,
semaphore: Arc<Semaphore>,
config: ProviderBodyOwnerConfig,
) {
let idle_timeout = config.idle_timeout;
let idle = sleep(idle_timeout);
tokio::pin!(idle);
while !cancel_token.is_cancelled() {
select! {
biased;
() = cancel_token.cancelled() => {
debug!("Provider body owner task observed cancellation signal");
break;
}
() = &mut idle => {
debug!("Provider body owner task idle timeout expired ({idle_timeout:?})");
if let Some(reason) = &close_reason {
let _ = reason.compare_exchange(
ProviderCloseReason::Unspecified as u8,
ProviderCloseReason::IdleTimeout as u8,
Ordering::AcqRel,
Ordering::Relaxed,
);
}
cancel_token.cancel();
break;
}
chunk = stream.next() => {
idle.as_mut().reset(Instant::now() + idle_timeout);
match chunk {
Some(Ok(bytes)) => {
let chunk_len = bytes.len();
let permits = chunk_len.min(config.max_buffer_bytes);
if permits > 0 {
let acquired = select! {
biased;
() = cancel_token.cancelled() => None,
permit = Arc::clone(&semaphore).acquire_many_owned(u32::try_from(permits).unwrap_or(u32::MAX)) => permit.ok(),
};
let Some(permit) = acquired else {
cancel_token.cancel();
break;
};
permit.forget();
}
let send_res = match tx.try_send(Ok(bytes)) {
Ok(()) => Ok(()),
Err(TrySendError::Full(item)) => {
select! {
biased;
() = cancel_token.cancelled() => Err(()),
res = tx.send(item) => res.map_err(|_| ()),
}
}
Err(TrySendError::Closed(_)) => Err(()),
};
if send_res.is_err() {
if permits > 0 {
semaphore.add_permits(permits);
}
if let Some(reason) = &close_reason {
let _ = reason.compare_exchange(
ProviderCloseReason::Unspecified as u8,
ProviderCloseReason::ClientClosed as u8,
Ordering::AcqRel,
Ordering::Relaxed,
);
}
cancel_token.cancel();
break;
}
}
Some(Err(err)) => {
if let Some(reason) = &close_reason {
let _ = reason.compare_exchange(
ProviderCloseReason::Unspecified as u8,
ProviderCloseReason::ProviderError as u8,
Ordering::AcqRel,
Ordering::Relaxed,
);
}
select! {
biased;
() = cancel_token.cancelled() => {},
_ = tx.send(Err(err)) => {},
}
break;
}
None => {
debug!("Provider body owner task completed cleanly on EOF");
break;
}
}
}
}
}
// Explicitly drop stream before signalling completion, releasing reqwest/hyper socket
drop(stream);
drop(tx);
if let Some(completion) = completion_token {
completion.cancel();
}
}
}
impl Stream for ProviderBodyOwner {
type Item = Result<Bytes, StreamError>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
match Pin::new(&mut this.stream).poll_next(cx) {
Poll::Ready(Some(Ok(bytes))) => {
this.semaphore.add_permits(bytes.len().min(this.max_buffer_bytes));
Poll::Ready(Some(Ok(bytes)))
}
Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))),
Poll::Ready(None) => Poll::Ready(None),
Poll::Pending => {
if this.close_cancelled.as_mut().poll(cx).is_ready() {
Poll::Ready(None)
} else {
Poll::Pending
}
}
}
}
}
impl Drop for ProviderBodyOwner {
fn drop(&mut self) { self.cancel_token.cancel(); }
}
#[cfg(test)]
mod tests {
use super::*;
use futures::Stream;
use tokio::sync::oneshot;
struct GatedDropProbeStream {
gate: oneshot::Receiver<()>,
dropped: Option<oneshot::Sender<()>>,
yielded: bool,
}
impl Stream for GatedDropProbeStream {
type Item = Result<Bytes, StreamError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
if self.yielded {
return Poll::Pending;
}
match Pin::new(&mut self.gate).poll(cx) {
Poll::Ready(Ok(())) => {
self.yielded = true;
Poll::Ready(Some(Ok(Bytes::from_static(b"chunk"))))
}
Poll::Ready(Err(_)) => Poll::Ready(None),
Poll::Pending => Poll::Pending,
}
}
}
impl Drop for GatedDropProbeStream {
fn drop(&mut self) {
if let Some(dropped) = self.dropped.take() {
let _ = dropped.send(());
}
}
}
#[tokio::test]
async fn unpolled_body_owner_terminates_upstream_when_cancelled() {
let (gate_tx, gate_rx) = oneshot::channel();
let (dropped_tx, dropped_rx) = oneshot::channel();
let cancel = CancellationToken::new();
let completion = CancellationToken::new();
let close_reason = Arc::new(AtomicU8::new(0));
let upstream = GatedDropProbeStream { gate: gate_rx, dropped: Some(dropped_tx), yielded: false };
let owner = ProviderBodyOwner::new(
Box::pin(upstream),
ProviderBodyOwnerConfig::for_direct_body(),
cancel.clone(),
Some(completion.clone()),
Some(close_reason.clone()),
);
// Cancel the owner WITHOUT EVER POLLING IT
cancel.cancel();
// Completion must fire
tokio::time::timeout(Duration::from_secs(1), completion.cancelled())
.await
.expect("completion token must be signalled when owner task terminates");
// Upstream stream must have been dropped
gate_tx.send(()).unwrap_err(); // gate_rx is dropped when upstream is dropped!
tokio::time::timeout(Duration::from_secs(1), dropped_rx)
.await
.expect("upstream stream must be dropped without downstream polling")
.expect("drop notification delivered");
drop(owner);
}
#[tokio::test]
async fn error_delivery_does_not_block_cancellation_when_channel_full() {
let (dropped_tx, dropped_rx) = oneshot::channel();
let cancel = CancellationToken::new();
let completion = CancellationToken::new();
let close_reason = Arc::new(AtomicU8::new(0));
struct TwoChunksThenErrStream {
count: usize,
dropped: Option<oneshot::Sender<()>>,
}
impl Stream for TwoChunksThenErrStream {
type Item = Result<Bytes, StreamError>;
fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
if self.count < 2 {
self.count += 1;
Poll::Ready(Some(Ok(Bytes::from_static(b"chunk"))))
} else if self.count == 2 {
self.count += 1;
Poll::Ready(Some(Err(StreamError::Stream("test error".to_owned()))))
} else {
Poll::Pending
}
}
}
impl Drop for TwoChunksThenErrStream {
fn drop(&mut self) {
if let Some(d) = self.dropped.take() {
let _ = d.send(());
}
}
}
let upstream = TwoChunksThenErrStream { count: 0, dropped: Some(dropped_tx) };
let owner = ProviderBodyOwner::new(
Box::pin(upstream),
ProviderBodyOwnerConfig::for_direct_body(),
cancel.clone(),
Some(completion.clone()),
Some(close_reason.clone()),
);
tokio::time::sleep(Duration::from_millis(50)).await;
cancel.cancel();
tokio::time::timeout(Duration::from_millis(500), completion.cancelled())
.await
.expect("completion token must be signalled when owner task terminates on error delivery");
tokio::time::timeout(Duration::from_millis(500), dropped_rx)
.await
.expect("upstream stream must be dropped")
.expect("drop notification delivered");
drop(owner);
}
}
bin/run-testkit-scenarios.sh
Executable
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd -P)"
cd "${REPO_ROOT}"
TARGET_DIR="${CARGO_TARGET_DIR:-${REPO_ROOT}/target}"
SUT_BINARY="${TARGET_DIR}/debug/tuliprox"
TESTKIT_BINARY="${TARGET_DIR}/debug/tuliprox-testkit"
REPORT_DIR="${REPO_ROOT}/testkit-report"
if [[ ! -x "${SUT_BINARY}" ]]; then
echo "❌ Tuliprox SUT binary not found at ${SUT_BINARY}. Build it first: cargo build -p tuliprox" >&2
exit 1
fi
if [[ ! -x "${TESTKIT_BINARY}" ]]; then
echo "❌ Testkit binary not found at ${TESTKIT_BINARY}. Build it first: cargo build -p tuliprox-testkit" >&2
exit 1
fi
mkdir -p "${REPORT_DIR}"
export TULIPROX_TESTKIT_SUT_BINARY="${SUT_BINARY}"
SUITE_RUN_ID="${TULIPROX_TESTKIT_RUN_ID:-$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || echo "suite-$(date +%s)")}"
FAILED_SCENARIOS=()
PASSED_COUNT=0
echo "🚀 Starting testkit scenario suite (run_id: ${SUITE_RUN_ID})..."
for scenario_path in test/fixtures/testkit/scenarios/*.yml; do
scenario_name="$(basename "${scenario_path}" .yml)"
scenario_dir="${REPORT_DIR}/${scenario_name}"
rm -rf "${scenario_dir}"
mkdir -p "${scenario_dir}"
echo " ▶ Running scenario: ${scenario_name}"
set +e
"${TESTKIT_BINARY}" controller --scenario "${scenario_path}" --report-directory "${scenario_dir}" --run-id "${SUITE_RUN_ID}" > "${scenario_dir}/scenario.log" 2>&1
exit_code=$?
set -e
if [[ ${exit_code} -eq 0 ]]; then
echo " ✅ ${scenario_name} PASSED"
PASSED_COUNT=$((PASSED_COUNT + 1))
else
echo " ❌ ${scenario_name} FAILED (exit code ${exit_code})"
FAILED_SCENARIOS+=("${scenario_name}")
if [[ ! -f "${scenario_dir}/run.json" ]]; then
cat <<EOF > "${scenario_dir}/run.json"
{
"schema_version": 1,
"scenario": "${scenario_name}",
"outcome": "failed",
"policy_hash": "pre-start",
"run_id": "${SUITE_RUN_ID}",
"error_kind": "controller_crashed",
"error_message": "Controller exited with code ${exit_code} before run.json could be generated",
"exit_code": ${exit_code},
"events": []
}
EOF
cat <<EOF > "${scenario_dir}/summary.txt"
scenario: ${scenario_name}
outcome: failed
policy: pre-start
run_id: ${SUITE_RUN_ID}
events: 0
EOF
touch "${scenario_dir}/events.ndjson"
cat <<EOF > "${scenario_dir}/junit.xml"
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="tuliprox-testkit" tests="1" failures="1"><testcase name="${scenario_name}"><failure message="failed"/></testcase></testsuite>
EOF
fi
cat "${scenario_dir}/scenario.log" >&2
fi
done
echo ""
echo "========================================"
echo "Testkit Scenario Summary:"
echo " Run ID: ${SUITE_RUN_ID}"
echo " Passed: ${PASSED_COUNT}"
echo " Failed: ${#FAILED_SCENARIOS[@]}"
echo "========================================"
FAILED_JSON_ARRAY="[]"
if [[ ${#FAILED_SCENARIOS[@]} -gt 0 ]]; then
FAILED_JSON_ARRAY="[$(printf '"%s",' "${FAILED_SCENARIOS[@]}" | sed 's/,$//')]"
fi
cat <<EOF > "${REPORT_DIR}/summary.json"
{
"run_id": "${SUITE_RUN_ID}",
"total": $((PASSED_COUNT + ${#FAILED_SCENARIOS[@]})),
"passed": ${PASSED_COUNT},
"failed": ${#FAILED_SCENARIOS[@]},
"failing_scenarios": ${FAILED_JSON_ARRAY}
}
EOF
if [[ ${#FAILED_SCENARIOS[@]} -gt 0 ]]; then
echo "❌ Failing scenarios: ${FAILED_SCENARIOS[*]}" >&2
exit 1
fi
echo "✅ All scenarios passed successfully!"
docker/Dockerfile
+6 -2
View File
@@ -35,15 +35,19 @@ COPY backend/parser/Cargo.toml ./backend/parser/
COPY backend/processing/Cargo.toml ./backend/processing/
COPY backend/repository/Cargo.toml ./backend/repository/
COPY backend/session/Cargo.toml ./backend/session/
COPY tools/tuliprox-testkit/Cargo.toml ./tools/tuliprox-testkit/
RUN mkdir -p frontend/src backend/app/src/tools && \
RUN mkdir -p frontend/src backend/app/src/tools tools/tuliprox-testkit/src && \
printf 'fn main() {}\n' > frontend/src/main.rs && \
printf 'fn main() {}\n' > backend/app/src/main.rs && \
printf 'fn main() {}\n' > backend/app/src/tools/flags_builder.rs && \
printf '\n' > tools/tuliprox-testkit/src/lib.rs && \
printf 'fn main() {}\n' > tools/tuliprox-testkit/src/main.rs && \
for crate in auth btree config-loader core curation dvr hls iptv library media-server metadata messaging mpegts parser processing repository session; do \
mkdir -p "backend/${crate}/src"; \
printf '\n' > "backend/${crate}/src/lib.rs"; \
done
done && \
cargo metadata --no-deps --format-version 1 > /dev/null
# -----------------------------------------------------------------
# Stage 1: Build the Rust binary for production
docker/cross.Dockerfile
+6 -2
View File
@@ -39,15 +39,19 @@ COPY backend/parser/Cargo.toml ./backend/parser/
COPY backend/processing/Cargo.toml ./backend/processing/
COPY backend/repository/Cargo.toml ./backend/repository/
COPY backend/session/Cargo.toml ./backend/session/
COPY tools/tuliprox-testkit/Cargo.toml ./tools/tuliprox-testkit/
RUN mkdir -p frontend/src backend/app/src/tools && \
RUN mkdir -p frontend/src backend/app/src/tools tools/tuliprox-testkit/src && \
printf 'fn main() {}\n' > frontend/src/main.rs && \
printf 'fn main() {}\n' > backend/app/src/main.rs && \
printf 'fn main() {}\n' > backend/app/src/tools/flags_builder.rs && \
printf '\n' > tools/tuliprox-testkit/src/lib.rs && \
printf 'fn main() {}\n' > tools/tuliprox-testkit/src/main.rs && \
for crate in auth btree config-loader core curation dvr hls iptv library media-server metadata messaging mpegts parser processing repository session; do \
mkdir -p "backend/${crate}/src"; \
printf '\n' > "backend/${crate}/src/lib.rs"; \
done
done && \
cargo metadata --no-deps --format-version 1 > /dev/null
# -----------------------------------------------------------------
# Stage 1: Build the Rust binary for production
test/fixtures/testkit/custom-video-fixtures.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
{
"schema_version": 1,
"fixtures": {
"hls/channel_unavailable.ts": "afefcdbcf3749e4e7a89321239dfcb7d2c0bdc4d2d2fb1cf4c883962c837c92f",
"hls/hls_session_or_lease_expired.ts": "9fe61499c2080f4aa1e362dfb72ec7c0a4b283f08d7dcac1f21590bd4db2fe55",
"hls/low_priority_preempted.ts": "9cc3a7b63f02e8e0bc45fe3b59c18f52443ded43fa0c9121fb0ea676def412a9",
"hls/panel_api_provisioning.ts": "b481b7b85f79883b41f8a1436301257ee5cb5a45e30993233ec73a2a5824ff13",
"hls/provider_connections_exhausted.ts": "099181ddcf9e7edc5007db6d801348afb132e27382eb4e0d33589dbe07cee790",
"hls/user_account_expired.ts": "110ea30bdec0f4d6457dcbfd2c8fb8dc0b2f517d8dedb473de1191d6de369464",
"hls/user_connections_exhausted.ts": "6ab07858d03e19b68eea8cbaecb67ad4d276633c8b423253b7f39c585cab73fe"
}
}
test/fixtures/testkit/scenarios/evict-same-ip-latest.yml
Vendored
+62
View File
@@ -0,0 +1,62 @@
schema_version: 1
name: evict-same-ip-latest
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin:
run_id: evict-same-ip-latest
actors:
- id: tv-a
agent: local
username: admission-user
user_agent: tuliprox-testkit/tv-a
client_ip: { mode: forwarded, value: 10.20.0.10 }
- id: tv-b
agent: local
username: admission-user
user_agent: tuliprox-testkit/tv-b
client_ip: { mode: forwarded, value: 10.20.0.10 }
- id: tv-c
agent: local
username: admission-user
user_agent: tuliprox-testkit/tv-c
client_ip: { mode: forwarded, value: 10.20.0.10 }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
live-18: { origin_marker: 18, protocol: xtream_ts }
live-19: { origin_marker: 19, protocol: xtream_ts }
steps:
- command_id: start-a
start:
actor: tv-a
playback_id: playback-a
session_group: playback-a
channel: live-17
await:
valid_frames: 5
- command_id: start-b
start:
actor: tv-b
playback_id: playback-b
session_group: playback-b
channel: live-18
await:
valid_frames: 5
- command_id: start-c
expect: streaming
start:
actor: tv-c
playback_id: playback-c
session_group: playback-c
channel: live-19
policy_contract:
user_access_control: true
users:
admission-user:
max_connections: 2
admission_strategies:
- evict_user_same_ip_latest
test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml
Vendored
+26
View File
@@ -0,0 +1,26 @@
schema_version: 1
name: evict-same-ip-oldest
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin: { run_id: evict-same-ip-oldest }
actors:
- { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } }
- { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-b, client_ip: { mode: forwarded, value: 10.20.0.10 } }
- { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-c, client_ip: { mode: forwarded, value: 10.20.0.10 } }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
live-18: { origin_marker: 18, protocol: xtream_ts }
live-19: { origin_marker: 19, protocol: xtream_ts }
steps:
- { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } }
- { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } }
- { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } }
policy_contract:
user_access_control: true
users: { admission-user: { max_connections: 2 } }
admission_strategies: [evict_user_same_ip_oldest]
test/fixtures/testkit/scenarios/evict-user-latest.yml
Vendored
+26
View File
@@ -0,0 +1,26 @@
schema_version: 1
name: evict-user-latest
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin: { run_id: evict-user-latest }
actors:
- { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } }
- { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-b, client_ip: { mode: forwarded, value: 10.20.0.11 } }
- { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-c, client_ip: { mode: forwarded, value: 10.20.0.12 } }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
live-18: { origin_marker: 18, protocol: xtream_ts }
live-19: { origin_marker: 19, protocol: xtream_ts }
steps:
- { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } }
- { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } }
- { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } }
policy_contract:
user_access_control: true
users: { admission-user: { max_connections: 2 } }
admission_strategies: [evict_user_latest]
test/fixtures/testkit/scenarios/evict-user-oldest.yml
Vendored
+26
View File
@@ -0,0 +1,26 @@
schema_version: 1
name: evict-user-oldest
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin: { run_id: evict-user-oldest }
actors:
- { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } }
- { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-b, client_ip: { mode: forwarded, value: 10.20.0.11 } }
- { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-c, client_ip: { mode: forwarded, value: 10.20.0.12 } }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
live-18: { origin_marker: 18, protocol: xtream_ts }
live-19: { origin_marker: 19, protocol: xtream_ts }
steps:
- { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } }
- { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } }
- { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } }
policy_contract:
user_access_control: true
users: { admission-user: { max_connections: 2 } }
admission_strategies: [evict_user_oldest]
test/fixtures/testkit/scenarios/hard-limit.yml
Vendored
+47
View File
@@ -0,0 +1,47 @@
schema_version: 1
name: hard-limit-denies-second-playback
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin:
run_id: hard-limit-denies-second-playback
actors:
- id: tv-a
agent: local
username: admission-user
user_agent: tuliprox-testkit/tv-a
client_ip: { mode: forwarded, value: 10.20.0.10 }
- id: tv-b
agent: local
username: admission-user
user_agent: tuliprox-testkit/tv-b
client_ip: { mode: forwarded, value: 10.20.0.10 }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
steps:
- command_id: start-a
start:
actor: tv-a
playback_id: playback-a
session_group: playback-a
channel: live-17
await:
valid_frames: 5
- command_id: start-b
expect: rejected
start:
actor: tv-b
playback_id: playback-b
session_group: playback-b
channel: live-17
policy_contract:
user_access_control: true
users:
admission-user:
max_connections: 1
soft_connections: 0
admission_strategies: []
test/fixtures/testkit/scenarios/provider-hard-limit.yml
Vendored
+38
View File
@@ -0,0 +1,38 @@
schema_version: 1
name: provider-hard-limit-denies-distinct-stream
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin:
run_id: provider-hard-limit-denies-distinct-stream
actors:
- id: tv-a
agent: local
username: admission-user
user_agent: tuliprox-testkit/provider-a
client_ip: { mode: forwarded, value: 10.20.0.10 }
- id: tv-b
agent: local
username: admission-user
user_agent: tuliprox-testkit/provider-b
client_ip: { mode: forwarded, value: 10.20.0.11 }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
live-18: { origin_marker: 18, protocol: xtream_ts }
steps:
- command_id: start-a
start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }
await: { valid_frames: 5 }
- command_id: start-b
expect: rejected
start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }
policy_contract:
user_access_control: true
users:
admission-user: { max_connections: 2 }
admission_strategies: []
provider_max_connections: 1
test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml
Vendored
+38
View File
@@ -0,0 +1,38 @@
schema_version: 1
name: shared-stream-uses-one-provider-slot
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin:
run_id: shared-stream-uses-one-provider-slot
actors:
- id: tv-a
agent: local
username: admission-user
user_agent: tuliprox-testkit/shared-a
client_ip: { mode: forwarded, value: 10.20.0.10 }
- id: tv-b
agent: local
username: admission-user
user_agent: tuliprox-testkit/shared-b
client_ip: { mode: forwarded, value: 10.20.0.11 }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
steps:
- command_id: start-a
start: { actor: tv-a, playback_id: playback-a, session_group: shared-live-17, channel: live-17 }
await: { valid_frames: 5 }
- command_id: start-b
start: { actor: tv-b, playback_id: playback-b, session_group: shared-live-17, channel: live-17 }
await: { valid_frames: 5 }
policy_contract:
user_access_control: true
users:
admission-user: { max_connections: 2 }
admission_strategies: []
provider_max_connections: 1
expected_provider_slots: 1
test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml
Vendored
+99
View File
@@ -0,0 +1,99 @@
schema_version: 1
name: vod-range-reopen-preserves-three-live
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin:
run_id: vod-range-reopen-preserves-three-live
account_limit: 5
limit_mode: reject_new
actors:
- id: live-viewer-1
agent: local
username: admission-user
user_agent: tuliprox-testkit/live-1
client_ip: { mode: forwarded, value: 10.20.0.10 }
- id: live-viewer-2
agent: local
username: admission-user
user_agent: tuliprox-testkit/live-2
client_ip: { mode: forwarded, value: 10.20.0.11 }
- id: live-viewer-3
agent: local
username: admission-user
user_agent: tuliprox-testkit/live-3
client_ip: { mode: forwarded, value: 10.20.0.12 }
- id: vod-viewer-ua1
agent: local
username: admission-user
user_agent: tuliprox-testkit/vod-player-ua1
client_ip: { mode: forwarded, value: 10.20.0.13 }
- id: vod-viewer-ua2
agent: local
username: admission-user
user_agent: tuliprox-testkit/vod-player-ua2
client_ip: { mode: forwarded, value: 10.20.0.13 }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
live-18: { origin_marker: 18, protocol: xtream_ts }
live-19: { origin_marker: 19, protocol: xtream_ts }
steps:
- command_id: start-live-1
start: { actor: live-viewer-1, playback_id: live-1, session_group: live-1, channel: live-17 }
await: { valid_frames: 3 }
- command_id: start-live-2
start: { actor: live-viewer-2, playback_id: live-2, session_group: live-2, channel: live-18 }
await: { valid_frames: 3 }
- command_id: start-live-3
start: { actor: live-viewer-3, playback_id: live-3, session_group: live-3, channel: live-19 }
await: { valid_frames: 3 }
assert_origin:
active_body: 3
max_active_body: 3
max_active_tcp: 3
- command_id: start-vod-probe-1
start:
actor: vod-viewer-ua1
playback_id: vod-probe-1
session_group: vod-movie
vod_object: movie.mkv
range: "bytes=0-524287"
read_limit_bytes: 32768
post_read_action: pause
assert_origin:
active_body: 4
max_active_body: 4
max_active_tcp: 4
- command_id: start-vod-probe-2
start:
actor: vod-viewer-ua1
playback_id: vod-probe-2
session_group: vod-movie
vod_object: movie.mkv
range: "bytes=1048576-1572863"
read_limit_bytes: 32768
post_read_action: pause
assert_origin:
active_body: 4
max_active_body: 4
max_active_tcp: 4
no_limit_rejections: true
no_evictions: true
- command_id: stop-vod-probe-2
stop: { playback_id: vod-probe-2 }
- command_id: stop-vod-probe-1
stop: { playback_id: vod-probe-1 }
assert_origin:
active_body: 3
max_active_body: 4
max_active_tcp: 4
policy_contract:
user_access_control: true
users:
admission-user: { max_connections: 10 }
admission_strategies: []
provider_max_connections: 4
test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml
Vendored
+94
View File
@@ -0,0 +1,94 @@
schema_version: 1
name: vod-range-reopen-strict-cap
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin:
run_id: vod-range-reopen-strict-cap
account_limit: 4
limit_mode: reject_new
actors:
- id: live-viewer-1
agent: local
username: admission-user
user_agent: tuliprox-testkit/live-1
client_ip: { mode: forwarded, value: 10.20.0.10 }
- id: live-viewer-2
agent: local
username: admission-user
user_agent: tuliprox-testkit/live-2
client_ip: { mode: forwarded, value: 10.20.0.11 }
- id: live-viewer-3
agent: local
username: admission-user
user_agent: tuliprox-testkit/live-3
client_ip: { mode: forwarded, value: 10.20.0.12 }
- id: vod-viewer-ua1
agent: local
username: admission-user
user_agent: tuliprox-testkit/vod-player-ua1
client_ip: { mode: forwarded, value: 10.20.0.13 }
channels:
live-17: { origin_marker: 17, protocol: xtream_ts }
live-18: { origin_marker: 18, protocol: xtream_ts }
live-19: { origin_marker: 19, protocol: xtream_ts }
steps:
- command_id: start-live-1
start: { actor: live-viewer-1, playback_id: live-1, session_group: live-1, channel: live-17 }
await: { valid_frames: 3 }
- command_id: start-live-2
start: { actor: live-viewer-2, playback_id: live-2, session_group: live-2, channel: live-18 }
await: { valid_frames: 3 }
- command_id: start-live-3
start: { actor: live-viewer-3, playback_id: live-3, session_group: live-3, channel: live-19 }
await: { valid_frames: 3 }
assert_origin:
active_body: 3
max_active_body: 3
max_active_tcp: 3
- command_id: start-vod-probe-1
start:
actor: vod-viewer-ua1
playback_id: vod-probe-1
session_group: vod-movie
vod_object: movie.mkv
range: "bytes=0-524287"
read_limit_bytes: 32768
post_read_action: pause
assert_origin:
active_body: 4
max_active_body: 4
max_active_tcp: 4
- command_id: start-vod-probe-2
start:
actor: vod-viewer-ua1
playback_id: vod-probe-2
session_group: vod-movie
vod_object: movie.mkv
range: "bytes=1048576-1572863"
read_limit_bytes: 32768
post_read_action: pause
assert_origin:
active_body: 4
max_active_body: 4
max_active_tcp: 4
no_limit_rejections: true
no_evictions: true
- command_id: stop-vod-probe-2
stop: { playback_id: vod-probe-2 }
- command_id: stop-vod-probe-1
stop: { playback_id: vod-probe-1 }
assert_origin:
active_body: 3
max_active_body: 4
max_active_tcp: 4
policy_contract:
user_access_control: true
users:
admission-user: { max_connections: 10 }
admission_strategies: []
provider_max_connections: 4
test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml
Vendored
+65
View File
@@ -0,0 +1,65 @@
schema_version: 1
name: vod-reopen-backpressured-body
tuliprox:
base_url: http://fixture.invalid
execution_mode: isolated_fixture
bootstrap:
command: "${env:TULIPROX_TESTKIT_SUT_BINARY}"
arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"]
readiness_url: "{base_url}/healthcheck"
origin:
run_id: vod-reopen-backpressured-body
account_limit: 2
limit_mode: reject_new
actors:
- id: vod-viewer
agent: local
username: admission-user
user_agent: tuliprox-testkit/backpressured-vod
client_ip: { mode: forwarded, value: 10.20.0.20 }
channels: {}
steps:
# First probe: read only 16 KiB out of 512 KiB, then pause to create downstream backpressure
- command_id: start-vod-backpressured
start:
actor: vod-viewer
playback_id: vod-bp-1
session_group: vod-bp
vod_object: movie.mkv
range: "bytes=0-524287"
read_limit_bytes: 16384
post_read_action: pause
assert_origin:
active_body: 1
max_active_body: 1
max_active_tcp: 1
# Second probe on the same session while first probe is still open and backpressured.
# Forced reopen must reclaim the single provider slot (max_connections: 1) without peak violations.
- command_id: start-vod-next
start:
actor: vod-viewer
playback_id: vod-bp-2
session_group: vod-bp
vod_object: movie.mkv
range: "bytes=1048576-1572863"
read_limit_bytes: 65536
post_read_action: pause
assert_origin:
active_body: 1
max_active_body: 1
max_active_tcp: 1
no_limit_rejections: true
- command_id: stop-vod-next
stop: { playback_id: vod-bp-2 }
- command_id: stop-vod-backpressured
stop: { playback_id: vod-bp-1 }
assert_origin:
active_body: 0
max_active_body: 1
max_active_tcp: 1
policy_contract:
user_access_control: true
users:
admission-user: { max_connections: 2 }
admission_strategies: []
provider_max_connections: 1
tools/tuliprox-testkit/Cargo.toml
+35
View File
@@ -0,0 +1,35 @@
[package]
name = "tuliprox-testkit"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[dependencies]
axum = { version = "0.8.9", features = ["macros", "default", "ws"] }
base64.workspace = true
blake3.workspace = true
bytes.workspace = true
clap = { version = "4.6.6", features = ["derive"] }
crc32fast = "1.5.1"
futures.workspace = true
reqwest = { version = "0.13.4", features = ["json", "stream", "rustls"] }
serde.workspace = true
serde_json.workspace = true
serde-saphyr.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "process", "signal", "sync", "time"] }
tokio-stream = "0.1.19"
tokio-tungstenite = "0.29.0"
tempfile = "3.27.0"
pin-project-lite = "0.2.17"
url.workspace = true
hyper = { version = "1.11.0", features = ["server", "http1"] }
hyper-util = { version = "0.1.20", features = ["server", "server-auto", "tokio"] }
tower = "0.5.3"
tower-service = "0.3.3"
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
tools/tuliprox-testkit/src/adaptive.rs
+80
View File
@@ -0,0 +1,80 @@
use crate::TestkitError;
use std::{collections::HashMap, time::Duration};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AdaptiveSessionState {
Prepared,
Active,
Preserved,
}
#[derive(Debug)]
struct Session {
state: AdaptiveSessionState,
expires_at: tokio::time::Instant,
}
#[derive(Debug, Default)]
pub struct AdaptiveSessionTracker {
sessions: HashMap<String, Session>,
}
impl AdaptiveSessionTracker {
pub fn manifest(&mut self, session_group: String, ttl: Duration) -> Result<AdaptiveSessionState, TestkitError> {
if self.sessions.contains_key(&session_group) {
return Err(TestkitError::Protocol("adaptive manifest created a duplicate logical session".to_owned()));
}
self.sessions.insert(
session_group,
Session { state: AdaptiveSessionState::Prepared, expires_at: tokio::time::Instant::now() + ttl },
);
Ok(AdaptiveSessionState::Prepared)
}
pub fn segment(&mut self, session_group: &str, ttl: Duration) -> Result<AdaptiveSessionState, TestkitError> {
let session = self.sessions.get_mut(session_group).ok_or_else(|| {
TestkitError::Configuration(format!("adaptive segment has no prepared session {session_group}"))
})?;
if tokio::time::Instant::now() > session.expires_at {
return Err(TestkitError::Protocol("adaptive session TTL expired before segment activation".to_owned()));
}
session.state = AdaptiveSessionState::Active;
session.expires_at = tokio::time::Instant::now() + ttl;
Ok(session.state)
}
pub fn preserve(&mut self, session_group: &str, ttl: Duration) -> Result<(), TestkitError> {
let session = self.sessions.get_mut(session_group).ok_or_else(|| {
TestkitError::Configuration(format!("cannot preserve unknown adaptive session {session_group}"))
})?;
session.state = AdaptiveSessionState::Preserved;
session.expires_at = tokio::time::Instant::now() + ttl;
Ok(())
}
pub fn reap_expired(&mut self) {
self.sessions.retain(|_, session| tokio::time::Instant::now() <= session.expires_at);
}
#[must_use]
pub fn state(&self, session_group: &str) -> Option<AdaptiveSessionState> {
self.sessions.get(session_group).map(|session| session.state)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(start_paused = true)]
async fn segment_activates_and_reacquisition_preserves_one_session() {
let mut tracker = AdaptiveSessionTracker::default();
assert_eq!(
tracker.manifest("hls-a".to_owned(), Duration::from_secs(10)).unwrap(),
AdaptiveSessionState::Prepared
);
assert_eq!(tracker.segment("hls-a", Duration::from_secs(10)).unwrap(), AdaptiveSessionState::Active);
tracker.preserve("hls-a", Duration::from_secs(10)).unwrap();
assert_eq!(tracker.segment("hls-a", Duration::from_secs(10)).unwrap(), AdaptiveSessionState::Active);
}
}
tools/tuliprox-testkit/src/bootstrap.rs
+462
View File
@@ -0,0 +1,462 @@
use crate::{
config::{BootstrapConfig, Channel, PolicyContract},
TestkitError,
};
use std::{
collections::HashMap,
fmt::Write,
net::SocketAddr,
path::{Path, PathBuf},
time::Duration,
};
use tempfile::TempDir;
use tokio::process::{Child, Command};
const FIXTURE_PASSWORD: &str = "testkit-password";
/// Files owned by one isolated run. They are written below a unique temporary
/// directory and must never refer to an operator's configuration or storage.
pub struct FixturePaths {
_directory: TempDir,
pub root: PathBuf,
pub config_file: PathBuf,
pub source_file: PathBuf,
pub api_proxy_file: PathBuf,
pub web_root: PathBuf,
}
impl FixturePaths {
pub fn create(
run_id: &str,
origin_address: SocketAddr,
tuliprox_address: SocketAddr,
policy: Option<&PolicyContract>,
channels: &HashMap<String, Channel>,
) -> Result<Self, TestkitError> {
let directory = tempfile::Builder::new().prefix("tuliprox-testkit-").tempdir()?;
let root = directory.path().to_path_buf();
let web_root = root.join("web");
let storage_dir = root.join("storage");
let backup_dir = root.join("backup");
let history_dir = root.join("history");
std::fs::create_dir_all(web_root.join("static/.well-known"))?;
std::fs::create_dir_all(&storage_dir)?;
std::fs::create_dir_all(&backup_dir)?;
std::fs::create_dir_all(&history_dir)?;
let config_file = root.join("config.yml");
let source_file = root.join("source.yml");
let api_proxy_file = root.join("api-proxy.yml");
std::fs::write(
&config_file,
render_config(tuliprox_address, &web_root, &storage_dir, &backup_dir, &history_dir, policy)?,
)?;
std::fs::write(&source_file, render_sources(run_id, origin_address, channels, policy))?;
std::fs::write(&api_proxy_file, render_api_proxy(tuliprox_address, policy))?;
Ok(Self { _directory: directory, root, config_file, source_file, api_proxy_file, web_root })
}
}
#[must_use]
pub const fn fixture_password() -> &'static str { FIXTURE_PASSWORD }
fn render_config(
tuliprox_address: SocketAddr,
web_root: &Path,
storage_dir: &Path,
backup_dir: &Path,
history_dir: &Path,
policy: Option<&PolicyContract>,
) -> Result<String, TestkitError> {
let user_access_control = policy.is_some_and(|contract| contract.user_access_control);
let grace = policy.and_then(|contract| contract.grace.as_ref());
let grace_millis = grace.map_or(0, |contract| contract.timeout_millis);
let grace_hold = grace.is_none_or(|contract| matches!(contract.mode, crate::oracle::GraceMode::HoldStream));
let mut config = format!(
"api:\n host: 127.0.0.1\n port: {}\n web_root: {}\nworking_dir: {}\nbackup_dir: {}\nuser_access_control: {user_access_control}\nupdate_on_boot: true\nlog:\n log_level: debug\nweb_ui:\n enabled: true\n user_ui_enabled: false\nreverse_proxy:\n stream:\n grace_period_millis: {grace_millis}\n grace_period_hold_stream: {grace_hold}\n shared_subscriber_idle_timeout_secs: 1\n",
tuliprox_address.port(),
yaml_scalar(web_root),
yaml_scalar(storage_dir),
yaml_scalar(backup_dir),
);
if let Some(policy) = policy {
if let Some(strategies) = &policy.admission_strategies {
if strategies.is_empty() {
config.push_str(" admission_strategies: []\n");
} else {
config.push_str(" admission_strategies:\n");
for strategy in strategies {
let encoded = serde_json::to_string(strategy).map_err(|error| {
TestkitError::Configuration(format!("cannot render admission strategy: {error}"))
})?;
let _ = writeln!(config, " - {}", encoded.trim_matches('\"'));
}
}
}
}
let _ = writeln!(
config,
" stream_history:\n stream_history_enabled: true\n stream_history_directory: {}\n rewrite_secret: 0123456789abcdef0123456789abcdef",
yaml_scalar(history_dir)
);
Ok(config)
}
fn render_sources(
run_id: &str,
origin_address: SocketAddr,
channels: &HashMap<String, Channel>,
policy: Option<&PolicyContract>,
) -> String {
let mut markers = channels.values().map(|channel| channel.origin_marker).collect::<Vec<_>>();
markers.sort_unstable();
markers.dedup();
if markers.is_empty() {
markers.push(17);
}
let marker_comment = markers.iter().map(u32::to_string).collect::<Vec<_>>().join(",");
let provider_limit = policy
.and_then(|contract| contract.provider_max_connections)
.map_or_else(String::new, |limit| format!(" max_connections: {limit}\n"));
format!(
"inputs:\n - name: testkit-origin\n type: m3u\n url: http://{origin_address}/catalog/input.m3u?run={run_id}\n{provider_limit}sources:\n - inputs:\n - testkit-origin\n targets:\n - name: testkit\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n mpeg_ts: true\n# Origin markers for this fixture: {marker_comment}\n"
)
}
fn render_api_proxy(tuliprox_address: SocketAddr, policy: Option<&PolicyContract>) -> String {
let mut output = format!(
"server:\n - name: testkit\n protocol: http\n host: 127.0.0.1\n port: \"{}\"\n timezone: UTC\n message: Tuliprox testkit\nuser:\n - target: testkit\n credentials:\n",
tuliprox_address.port()
);
if let Some(policy) = policy {
for (username, user) in &policy.users {
let _ = writeln!(
output,
" - username: {}\n password: {}\n proxy: reverse\n server: testkit\n max_connections: {}\n soft_connections: {}\n status: Active",
yaml_string(username),
FIXTURE_PASSWORD,
user.max_connections,
user.soft_connections,
);
}
}
if !output.contains(" - username:") {
output.push_str(" - username: testkit\n password: testkit-password\n proxy: reverse\n server: testkit\n status: Active\n");
}
output
}
fn yaml_scalar(path: &Path) -> String { yaml_string(&path.to_string_lossy()) }
fn yaml_string(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) }
/// Owns only the SUT process launched for this run and waits for its explicit readiness endpoint.
pub struct IsolatedSut {
child: Child,
}
/// Owns every process and every generated file used by one self-contained
/// Tuliprox scenario. The fixture never adopts an externally running process.
pub struct IsolatedFixture {
pub paths: FixturePaths,
pub base_url: String,
pub origin_control_url: String,
origin: Child,
sut: IsolatedSut,
}
impl IsolatedFixture {
#[allow(clippy::too_many_arguments)]
pub async fn start(
bootstrap: &BootstrapConfig,
working_directory: &Path,
testkit_binary: &Path,
run_id: &str,
origin_config: Option<&crate::config::OriginConfig>,
policy: Option<&PolicyContract>,
channels: &HashMap<String, Channel>,
) -> Result<Self, TestkitError> {
const MAX_START_ATTEMPTS: usize = 5;
let mut last_error = None;
for attempt in 1..=MAX_START_ATTEMPTS {
let attempt_run_id = if attempt == 1 { run_id.to_string() } else { format!("{run_id}-retry{attempt}") };
match Self::start_attempt(
bootstrap,
working_directory,
testkit_binary,
&attempt_run_id,
origin_config,
policy,
channels,
)
.await
{
Ok(env) => {
if attempt > 1 {
eprintln!("Bootstrap succeeded on attempt {attempt}/{MAX_START_ATTEMPTS}");
}
return Ok(env);
}
Err(error) => {
eprintln!("Bootstrap start attempt {attempt}/{MAX_START_ATTEMPTS} failed: {error}; retrying with fresh ports...");
last_error = Some(error);
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
}
Err(last_error.unwrap_or_else(|| TestkitError::Protocol("failed to bootstrap environment".to_owned())))
}
#[allow(clippy::too_many_arguments)]
async fn start_attempt(
bootstrap: &BootstrapConfig,
working_directory: &Path,
testkit_binary: &Path,
run_id: &str,
origin_config: Option<&crate::config::OriginConfig>,
policy: Option<&PolicyContract>,
channels: &HashMap<String, Channel>,
) -> Result<Self, TestkitError> {
let origin_address = reserve_loopback_address()?;
let origin_control_address = reserve_loopback_address()?;
let tuliprox_address = reserve_loopback_address()?;
let paths = FixturePaths::create(run_id, origin_address, tuliprox_address, policy, channels)?;
let markers = fixture_markers(channels);
let mut origin_cmd = Command::new(testkit_binary);
origin_cmd
.arg("origin")
.arg("--listen")
.arg(origin_address.to_string())
.arg("--control-listen")
.arg(origin_control_address.to_string())
.arg("--run-id")
.arg(run_id)
.arg("--markers")
.arg(markers);
if let Some(origin_cfg) = origin_config {
if let Some(limit) = origin_cfg.account_limit {
origin_cmd.arg("--account-limit").arg(limit.to_string());
}
if let Some(mode) = origin_cfg.limit_mode {
let mode_str = match mode {
crate::origin_transport::OriginLimitMode::ObserveOnly => "observe_only",
crate::origin_transport::OriginLimitMode::RejectNew => "reject_new",
crate::origin_transport::OriginLimitMode::EvictOldest => "evict_oldest",
};
origin_cmd.arg("--limit-mode").arg(mode_str);
}
}
let mut origin = origin_cmd.current_dir(working_directory).spawn()?;
let origin_health = format!("http://{origin_control_address}/health");
if let Err(error) = wait_for_http_ready(&mut origin, &origin_health, Duration::from_secs(10)).await {
let _ = stop_child(&mut origin).await;
return Err(error);
}
let base_url = format!("http://{tuliprox_address}");
let rendered = render_bootstrap(bootstrap, &paths, &base_url)?;
let sut = match IsolatedSut::start(&rendered, working_directory).await {
Ok(sut) => sut,
Err(error) => {
let _ = stop_child(&mut origin).await;
return Err(error);
}
};
let origin_control_url = format!("http://{origin_control_address}");
Ok(Self { paths, base_url, origin_control_url, origin, sut })
}
pub async fn stop(mut self) -> Result<(), TestkitError> {
let sut_result = self.sut.stop().await;
let origin_result = stop_child(&mut self.origin).await;
match (sut_result, origin_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
(Err(sut_error), Err(origin_error)) => Err(TestkitError::Protocol(format!(
"Tuliprox cleanup failed ({sut_error}); origin cleanup failed ({origin_error})"
))),
}
}
}
fn reserve_loopback_address() -> Result<SocketAddr, TestkitError> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
let address = listener.local_addr()?;
drop(listener);
Ok(address)
}
fn fixture_markers(channels: &HashMap<String, Channel>) -> String {
let mut markers = channels.values().map(|channel| channel.origin_marker).collect::<Vec<_>>();
markers.sort_unstable();
markers.dedup();
if markers.is_empty() {
markers.push(17);
}
markers.iter().map(u32::to_string).collect::<Vec<_>>().join(",")
}
fn render_bootstrap(
bootstrap: &BootstrapConfig,
paths: &FixturePaths,
base_url: &str,
) -> Result<BootstrapConfig, TestkitError> {
let render = |value: &str| {
value
.replace("{config_file}", &paths.config_file.to_string_lossy())
.replace("{source_file}", &paths.source_file.to_string_lossy())
.replace("{api_proxy_file}", &paths.api_proxy_file.to_string_lossy())
.replace("{run_directory}", &paths.root.to_string_lossy())
.replace("{base_url}", base_url)
};
let command = resolve_environment_command(&render(&bootstrap.command))?;
Ok(BootstrapConfig {
command,
arguments: bootstrap.arguments.iter().map(|argument| render(argument)).collect(),
readiness_url: render(&bootstrap.readiness_url),
readiness_timeout_millis: bootstrap.readiness_timeout_millis,
})
}
fn resolve_environment_command(command: &str) -> Result<String, TestkitError> {
let Some(variable) = command.strip_prefix("${env:").and_then(|value| value.strip_suffix('}')) else {
return Ok(command.to_owned());
};
std::env::var(variable).map_err(|_| {
TestkitError::Configuration(format!("bootstrap command environment variable {variable} is not set"))
})
}
async fn wait_for_http_ready(child: &mut Child, readiness_url: &str, timeout: Duration) -> Result<(), TestkitError> {
let deadline = tokio::time::Instant::now() + timeout;
let client = reqwest::Client::new();
loop {
if let Some(status) = child.try_wait()? {
return Err(TestkitError::Protocol(format!("fixture process exited before readiness: {status}")));
}
if client.get(readiness_url).send().await.is_ok_and(|response| response.status().is_success()) {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(TestkitError::Protocol(format!("fixture process did not become ready at {readiness_url}")));
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
async fn stop_child(child: &mut Child) -> Result<(), TestkitError> {
if child.try_wait()?.is_some() {
return Ok(());
}
child.start_kill()?;
tokio::time::timeout(Duration::from_secs(5), child.wait())
.await
.map_err(|_| TestkitError::Protocol("fixture process did not exit during cleanup".to_owned()))??;
Ok(())
}
impl IsolatedSut {
pub async fn start(config: &BootstrapConfig, working_directory: &Path) -> Result<Self, TestkitError> {
if config.command.trim().is_empty() || config.readiness_url.trim().is_empty() {
return Err(TestkitError::Configuration("bootstrap command and readiness URL are required".to_owned()));
}
let child = Command::new(&config.command).args(&config.arguments).current_dir(working_directory).spawn()?;
let mut sut = Self { child };
if let Err(startup_error) =
sut.wait_ready(&config.readiness_url, Duration::from_millis(config.readiness_timeout_millis.max(1))).await
{
if let Err(cleanup_error) = sut.stop().await {
return Err(TestkitError::Protocol(format!(
"isolated Tuliprox startup failed ({startup_error}) and cleanup failed ({cleanup_error})"
)));
}
return Err(startup_error);
}
Ok(sut)
}
async fn wait_ready(&mut self, readiness_url: &str, timeout: Duration) -> Result<(), TestkitError> {
let deadline = tokio::time::Instant::now() + timeout;
let client = reqwest::Client::new();
loop {
if let Some(status) = self.child.try_wait()? {
return Err(TestkitError::Protocol(format!(
"isolated Tuliprox process exited before readiness: {status}"
)));
}
if let Ok(response) = client.get(readiness_url).send().await {
if response.status().is_success() {
return Ok(());
}
}
if tokio::time::Instant::now() >= deadline {
return Err(TestkitError::Protocol("isolated Tuliprox process did not become ready".to_owned()));
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
pub async fn stop(mut self) -> Result<(), TestkitError> {
if self.child.try_wait()?.is_none() {
self.child.start_kill()?;
self.child.wait().await?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn early_process_exit_does_not_wait_for_the_readiness_deadline() {
let config = BootstrapConfig {
command: "/bin/false".to_owned(),
arguments: Vec::new(),
readiness_url: "http://127.0.0.1:1/ready".to_owned(),
readiness_timeout_millis: 5_000,
};
let started_at = tokio::time::Instant::now();
let result = IsolatedSut::start(&config, Path::new("/tmp")).await;
assert!(matches!(result, Err(TestkitError::Protocol(message)) if message.contains("exited before readiness")));
assert!(started_at.elapsed() < Duration::from_secs(1));
}
#[test]
fn fixture_files_are_run_scoped_and_contain_no_operator_paths() -> Result<(), TestkitError> {
let policy = PolicyContract {
user_access_control: true,
users: HashMap::from([(
"fixture-user".to_owned(),
crate::config::UserPolicy { max_connections: 1, soft_connections: 0 },
)]),
admission_strategies: Some(vec![crate::oracle::AdmissionStrategy::EvictUserSameIpLatest]),
grace: None,
provider_max_connections: None,
expected_provider_slots: None,
};
let channels = HashMap::from([(
"fixture-channel".to_owned(),
Channel { origin_marker: 17, protocol: "xtream_ts".to_owned() },
)]);
let fixture = FixturePaths::create(
"fixture-run",
"127.0.0.1:9910".parse::<SocketAddr>().map_err(|error| TestkitError::Configuration(error.to_string()))?,
"127.0.0.1:8901".parse::<SocketAddr>().map_err(|error| TestkitError::Configuration(error.to_string()))?,
Some(&policy),
&channels,
)?;
let config = std::fs::read_to_string(&fixture.config_file)?;
let source = std::fs::read_to_string(&fixture.source_file)?;
let api_proxy = std::fs::read_to_string(&fixture.api_proxy_file)?;
assert!(config.contains(fixture.root.to_string_lossy().as_ref()));
assert!(source.contains("fixture-run"));
assert!(api_proxy.contains("fixture-user"));
assert!(!config.contains("/home/"));
Ok(())
}
}
tools/tuliprox-testkit/src/config.rs
+607
View File
@@ -0,0 +1,607 @@
use crate::{
oracle::{AdmissionStrategy, GraceMode},
TestkitError,
};
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, HashSet},
fs,
path::Path,
time::Duration,
};
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Scenario {
pub schema_version: u16,
pub name: String,
pub tuliprox: TuliproxConfig,
#[serde(default)]
pub origin: Option<OriginConfig>,
#[serde(default)]
pub actors: Vec<Actor>,
#[serde(default)]
pub channels: HashMap<String, Channel>,
#[serde(default)]
pub agents: AgentsConfig,
#[serde(default)]
pub policy_contract: Option<PolicyContract>,
#[serde(default)]
pub steps: Vec<Step>,
}
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AgentsConfig {
#[serde(default)]
pub required: Vec<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PolicyContract {
pub user_access_control: bool,
#[serde(default)]
pub users: HashMap<String, UserPolicy>,
/// `None` preserves an omitted field, which has distinct Tuliprox semantics
/// from an explicit empty admission-strategy list.
#[serde(default)]
pub admission_strategies: Option<Vec<AdmissionStrategy>>,
#[serde(default)]
pub grace: Option<GraceContract>,
#[serde(default)]
pub provider_max_connections: Option<u16>,
#[serde(default)]
pub expected_provider_slots: Option<usize>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct UserPolicy {
pub max_connections: usize,
#[serde(default)]
pub soft_connections: usize,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GraceContract {
pub mode: GraceMode,
pub timeout_millis: u64,
}
impl PolicyContract {
#[must_use]
pub fn effective_admission_strategies(&self) -> Vec<AdmissionStrategy> {
self.admission_strategies.clone().unwrap_or_else(|| {
self.grace.as_ref().map_or_else(Vec::new, |grace| match grace.mode {
GraceMode::Instant => vec![AdmissionStrategy::GraceInstantStream],
GraceMode::HoldStream => vec![AdmissionStrategy::GraceHoldStream],
})
})
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TuliproxConfig {
pub base_url: String,
#[serde(default)]
pub api_base_url: Option<String>,
#[serde(default)]
pub admin_username: Option<String>,
#[serde(default)]
pub admin_password_env: Option<String>,
#[serde(default)]
pub playlist_url: Option<String>,
#[serde(default)]
pub bootstrap: Option<BootstrapConfig>,
#[serde(default)]
pub execution_mode: ExecutionMode,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OriginConfig {
pub run_id: String,
#[serde(default)]
pub account_limit: Option<usize>,
#[serde(default)]
pub limit_mode: Option<crate::origin_transport::OriginLimitMode>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BootstrapConfig {
pub command: String,
#[serde(default)]
pub arguments: Vec<String>,
pub readiness_url: String,
#[serde(default = "default_readiness_timeout_millis")]
pub readiness_timeout_millis: u64,
}
fn default_readiness_timeout_millis() -> u64 { 30_000 }
#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ExecutionMode {
#[default]
IsolatedFixture,
ExistingInstance,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Actor {
pub id: String,
pub agent: String,
#[serde(default)]
pub username: Option<String>,
#[serde(default)]
pub password_env: Option<String>,
#[serde(default)]
pub user_agent: Option<String>,
#[serde(default)]
pub client_ip: Option<ClientIp>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ClientIp {
pub mode: String,
#[serde(default)]
pub value: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Channel {
pub origin_marker: u32,
pub protocol: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Step {
pub command_id: String,
#[serde(default)]
pub expect: ExpectedPlayback,
pub start: Option<Start>,
#[serde(default)]
pub stop: Option<Stop>,
#[serde(default)]
pub repeat: Option<Repeat>,
#[serde(default)]
pub await_frames: Option<u64>,
#[serde(default, rename = "await")]
pub await_condition: Option<AwaitCondition>,
#[serde(default)]
pub assert_origin: Option<AssertOrigin>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct ExpectedRejection {
#[serde(default)]
pub custom_video: Option<crate::custom_video::CustomVideoKind>,
#[serde(default)]
pub http_status: Option<u16>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ExpectedPlayback {
#[default]
Streaming,
Rejected,
RejectedWith(ExpectedRejection),
}
impl ExpectedPlayback {
#[must_use]
pub fn is_streaming(&self) -> bool { matches!(self, Self::Streaming) }
#[must_use]
pub fn is_rejected(&self) -> bool { matches!(self, Self::Rejected | Self::RejectedWith(_)) }
#[must_use]
pub fn matches_outcome(&self, outcome: &crate::protocol::PlaybackOutcome) -> bool {
use crate::protocol::{PlaybackOutcome, RejectionReason};
match (self, outcome) {
(Self::Streaming, PlaybackOutcome::Streaming { .. })
| (Self::Rejected, PlaybackOutcome::AdmissionRejected { .. }) => true,
(Self::RejectedWith(expected), PlaybackOutcome::AdmissionRejected { reason }) => match reason {
RejectionReason::CustomVideo(kind) => {
expected.custom_video.is_none_or(|expected_kind| expected_kind == *kind)
}
RejectionReason::HttpStatus(status) => {
expected.http_status.is_none_or(|expected_status| expected_status == *status)
}
_ => expected.custom_video.is_none() && expected.http_status.is_none(),
},
_ => false,
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AwaitCondition {
#[serde(default)]
pub valid_frames: Option<u64>,
}
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(deny_unknown_fields)]
pub struct AssertOrigin {
#[serde(default)]
pub max_active_tcp: Option<usize>,
#[serde(default)]
pub max_active_body: Option<usize>,
#[serde(default)]
pub active_body: Option<usize>,
#[serde(default)]
pub no_evictions: Option<bool>,
#[serde(default)]
pub no_limit_rejections: Option<bool>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum PostReadAction {
#[default]
KeepOpen,
Pause,
Close,
}
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(deny_unknown_fields)]
pub struct Start {
pub actor: String,
pub playback_id: String,
pub session_group: String,
#[serde(default)]
pub url: Option<String>,
#[serde(default)]
pub channel: Option<String>,
#[serde(default)]
pub method: Option<String>,
#[serde(default)]
pub range: Option<String>,
#[serde(default)]
pub expected_status: Option<u16>,
#[serde(default)]
pub expected_content_range: Option<String>,
#[serde(default)]
pub read_limit_bytes: Option<u64>,
#[serde(default)]
pub post_read_action: Option<PostReadAction>,
#[serde(default)]
pub user_agent: Option<String>,
#[serde(default)]
pub vod_object: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Stop {
pub playback_id: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Repeat {
pub count: usize,
pub steps: Vec<Step>,
}
impl Scenario {
pub fn from_path(path: &Path) -> Result<Self, TestkitError> {
let content = fs::read_to_string(path)?;
let scenario: Self =
serde_saphyr::from_str(&content).map_err(|error| TestkitError::Configuration(error.to_string()))?;
scenario.validate()?;
Ok(scenario)
}
pub fn validate(&self) -> Result<(), TestkitError> {
if self.schema_version != 1 {
return Err(TestkitError::Configuration("unsupported scenario schema version".to_owned()));
}
if self.name.trim().is_empty() || self.tuliprox.base_url.trim().is_empty() {
return Err(TestkitError::Configuration("scenario name and Tuliprox URL are required".to_owned()));
}
if self.tuliprox.execution_mode == ExecutionMode::IsolatedFixture && self.tuliprox.bootstrap.is_none() {
return Err(TestkitError::Configuration("isolated_fixture requires a bootstrap definition".to_owned()));
}
if self.origin.as_ref().is_some_and(|origin| origin.run_id.trim().is_empty()) {
return Err(TestkitError::Configuration("origin run_id cannot be empty".to_owned()));
}
if self.policy_contract.as_ref().and_then(|contract| contract.provider_max_connections) == Some(0) {
return Err(TestkitError::Configuration(
"provider_max_connections must be positive when specified".to_owned(),
));
}
let actor_ids = self.actors.iter().map(|actor| actor.id.as_str()).collect::<HashSet<_>>();
if actor_ids.len() != self.actors.len() || actor_ids.iter().any(|id| id.is_empty()) {
return Err(TestkitError::Configuration("actor IDs must be unique and non-empty".to_owned()));
}
let steps = self.expanded_steps()?;
let mut command_ids = HashSet::new();
let mut playback_ids = HashSet::new();
let mut seen_playback_ids = HashSet::new();
for step in &steps {
if !command_ids.insert(step.command_id.as_str()) || step.command_id.is_empty() {
return Err(TestkitError::Configuration("command IDs must be unique and non-empty".to_owned()));
}
if step.start.is_some() == step.stop.is_some() {
return Err(TestkitError::Configuration(
"each step must contain exactly one of start or stop".to_owned(),
));
}
let Some(start) = &step.start else {
let stop = step.stop.as_ref().ok_or_else(|| TestkitError::Configuration("missing stop".to_owned()))?;
if stop.playback_id.is_empty() || !playback_ids.contains(stop.playback_id.as_str()) {
return Err(TestkitError::Configuration(format!(
"stop references inactive playback {}",
stop.playback_id
)));
}
playback_ids.remove(stop.playback_id.as_str());
continue;
};
if !actor_ids.contains(start.actor.as_str()) {
return Err(TestkitError::Configuration(format!("unknown actor {}", start.actor)));
}
if start.playback_id.is_empty() || start.session_group.is_empty() {
return Err(TestkitError::Configuration("start requires playback_id and session_group".to_owned()));
}
let target_count = [start.url.is_some(), start.channel.is_some(), start.vod_object.is_some()]
.into_iter()
.filter(|&v| v)
.count();
if target_count != 1 {
return Err(TestkitError::Configuration(
"start requires exactly one of url, channel, or vod_object".to_owned(),
));
}
if start.url.as_deref().is_some_and(str::is_empty)
|| start.channel.as_deref().is_some_and(str::is_empty)
|| start.vod_object.as_deref().is_some_and(str::is_empty)
{
return Err(TestkitError::Configuration(
"start URL, channel, or vod_object cannot be empty".to_owned(),
));
}
if let Some(channel) = &start.channel {
if !self.channels.contains_key(channel) {
return Err(TestkitError::Configuration(format!("unknown channel {channel}")));
}
}
if !seen_playback_ids.insert(start.playback_id.as_str()) {
return Err(TestkitError::Configuration(format!("duplicate playback ID {}", start.playback_id)));
}
playback_ids.insert(start.playback_id.as_str());
if step.expect == ExpectedPlayback::Rejected
&& step
.await_frames
.or_else(|| step.await_condition.as_ref().and_then(|condition| condition.valid_frames))
.is_some()
{
return Err(TestkitError::Configuration("rejected playback must not await valid frames".to_owned()));
}
}
if let Some(contract) = &self.policy_contract {
if !contract.user_access_control && !contract.users.is_empty() {
return Err(TestkitError::Configuration("policy users require user_access_control".to_owned()));
}
if contract.grace.as_ref().is_some_and(|grace| grace.timeout_millis == 0) {
return Err(TestkitError::Configuration("grace timeout must be non-zero".to_owned()));
}
}
Ok(())
}
pub fn expanded_steps(&self) -> Result<Vec<Step>, TestkitError> {
let mut expanded = Vec::new();
expand_steps(&self.steps, "", &mut expanded)?;
Ok(expanded)
}
}
fn expand_steps(steps: &[Step], prefix: &str, expanded: &mut Vec<Step>) -> Result<(), TestkitError> {
const MAX_REPEAT_COUNT: usize = 1_000;
for step in steps {
if step.command_id.is_empty() {
return Err(TestkitError::Configuration("command IDs must be unique and non-empty".to_owned()));
}
if let Some(repeat) = &step.repeat {
if step.start.is_some() || step.stop.is_some() {
return Err(TestkitError::Configuration("repeat cannot be combined with start or stop".to_owned()));
}
if repeat.count == 0 || repeat.count > MAX_REPEAT_COUNT || repeat.steps.is_empty() {
return Err(TestkitError::Configuration("repeat requires 1 through 1000 child steps".to_owned()));
}
for iteration in 0..repeat.count {
expand_steps(&repeat.steps, &format!("{prefix}{}:{iteration}:", step.command_id), expanded)?;
}
continue;
}
if step.start.is_some() == step.stop.is_some() {
return Err(TestkitError::Configuration("each step must contain exactly one of start or stop".to_owned()));
}
let mut child = step.clone();
child.command_id = format!("{prefix}{}", step.command_id);
if let Some(start) = child.start.as_mut() {
start.playback_id = format!("{prefix}{}", start.playback_id);
start.session_group = format!("{prefix}{}", start.session_group);
}
if let Some(stop) = child.stop.as_mut() {
stop.playback_id = format!("{prefix}{}", stop.playback_id);
}
expanded.push(child);
}
Ok(())
}
#[must_use]
pub fn lease_duration(millis: u64) -> Duration { Duration::from_millis(millis.max(1)) }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn duplicate_playbacks_are_rejected() {
let scenario = Scenario {
schema_version: 1,
name: "test".to_owned(),
tuliprox: TuliproxConfig {
base_url: "http://example.invalid".to_owned(),
api_base_url: None,
admin_username: None,
admin_password_env: None,
playlist_url: None,
bootstrap: None,
execution_mode: ExecutionMode::IsolatedFixture,
},
origin: None,
actors: vec![Actor {
id: "a".to_owned(),
agent: "local".to_owned(),
username: None,
password_env: None,
user_agent: None,
client_ip: None,
}],
channels: HashMap::new(),
agents: AgentsConfig::default(),
policy_contract: None,
steps: (0..2)
.map(|number| Step {
command_id: number.to_string(),
expect: ExpectedPlayback::Streaming,
start: Some(Start {
actor: "a".to_owned(),
playback_id: "same".to_owned(),
session_group: "s".to_owned(),
url: Some("http://example.invalid".to_owned()),
channel: None,
..Default::default()
}),
stop: None,
repeat: None,
await_frames: None,
await_condition: None,
assert_origin: None,
})
.collect(),
};
assert!(scenario.validate().is_err());
}
#[test]
fn checked_in_admission_scenarios_match_the_versioned_schema() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test/fixtures/testkit/scenarios");
for filename in [
"hard-limit.yml",
"evict-same-ip-latest.yml",
"evict-same-ip-oldest.yml",
"evict-user-oldest.yml",
"evict-user-latest.yml",
"provider-hard-limit.yml",
"shared-stream-single-provider-slot.yml",
"vod-range-reopen-preserves-three-live.yml",
"vod-range-reopen-strict-cap.yml",
"vod-reopen-backpressured-body.yml",
] {
assert!(Scenario::from_path(&root.join(filename)).is_ok(), "{filename}");
}
}
#[test]
fn stop_must_reference_an_active_playback() {
let scenario: Scenario = serde_saphyr::from_str(
r"
schema_version: 1
name: stop-validation
tuliprox: { base_url: http://example.invalid, execution_mode: existing_instance }
actors: [{ id: a, agent: local }]
steps:
- command_id: stop-missing
stop: { playback_id: missing }
",
)
.unwrap();
assert!(scenario.validate().is_err());
}
#[test]
fn repeat_derives_distinct_playback_and_command_ids_per_iteration() {
let scenario: Scenario = serde_saphyr::from_str(
r"
schema_version: 1
name: repeat-validation
tuliprox: { base_url: http://example.invalid, execution_mode: existing_instance }
actors: [{ id: a, agent: local }]
steps:
- command_id: repeat
repeat:
count: 2
steps:
- command_id: begin
start: { actor: a, playback_id: playback, session_group: session, url: http://example.invalid/17.ts }
- command_id: end
stop: { playback_id: playback }
",
)
.unwrap();
assert!(scenario.validate().is_ok());
let expanded = scenario.expanded_steps().unwrap();
assert_eq!(expanded.len(), 4);
assert_eq!(expanded[0].command_id, "repeat:0:begin");
assert_eq!(expanded[0].start.as_ref().map(|start| start.playback_id.as_str()), Some("repeat:0:playback"));
assert_eq!(expanded[2].command_id, "repeat:1:begin");
assert_eq!(expanded[3].stop.as_ref().map(|stop| stop.playback_id.as_str()), Some("repeat:1:playback"));
}
#[test]
fn rejected_playback_does_not_match_500_corrupt_data_or_early_eof() {
use crate::{
custom_video::CustomVideoKind,
protocol::{PlaybackOutcome, RejectionReason},
};
let expected_rejected = ExpectedPlayback::Rejected;
// Proved admission rejection matches
let valid_cv = PlaybackOutcome::AdmissionRejected {
reason: RejectionReason::CustomVideo(CustomVideoKind::UserConnectionsExhausted),
};
assert!(expected_rejected.matches_outcome(&valid_cv));
let valid_http = PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(429) };
assert!(expected_rejected.matches_outcome(&valid_http));
// HTTP 500 MUST NOT pass as expected rejection
let http_500 = PlaybackOutcome::HttpError { status: 500 };
assert!(!expected_rejected.matches_outcome(&http_500));
// Invalid marker/corrupt frame MUST NOT pass
let invalid_marker = PlaybackOutcome::InvalidData { message: "wrong marker".to_owned() };
assert!(!expected_rejected.matches_outcome(&invalid_marker));
// Early EOF MUST NOT pass
let early_eof = PlaybackOutcome::UnexpectedEof { frames: 0, bytes: 0 };
assert!(!expected_rejected.matches_outcome(&early_eof));
// Transport error MUST NOT pass
let transport_err = PlaybackOutcome::TransportError { message: "connection reset".to_owned() };
assert!(!expected_rejected.matches_outcome(&transport_err));
// Idle timeout MUST NOT pass
assert!(!expected_rejected.matches_outcome(&PlaybackOutcome::IdleTimeout));
}
}
tools/tuliprox-testkit/src/control.rs
+63
View File
@@ -0,0 +1,63 @@
use crate::{
protocol::{PlaybackId, RunId},
TestkitError,
};
use std::collections::HashMap;
/// Tracks the semantic start identity for a run independently from transport message IDs.
/// A replay with identical parameters is harmless; a conflicting replay is rejected.
#[derive(Debug, Default)]
pub struct PlaybackRegistry {
starts: HashMap<(RunId, u64, PlaybackId), String>,
}
impl PlaybackRegistry {
pub fn register_start(
&mut self,
run_id: RunId,
generation: u64,
playback_id: PlaybackId,
fingerprint: String,
) -> Result<StartDisposition, TestkitError> {
let key = (run_id, generation, playback_id);
match self.starts.get(&key) {
None => {
self.starts.insert(key, fingerprint);
Ok(StartDisposition::New)
}
Some(existing) if existing == &fingerprint => Ok(StartDisposition::Replay),
Some(_) => Err(TestkitError::Protocol("playback ID was reused with different start parameters".to_owned())),
}
}
pub fn stop(&mut self, run_id: &RunId, generation: u64, playback_id: &PlaybackId) -> bool {
self.starts.remove(&(run_id.clone(), generation, playback_id.clone())).is_some()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StartDisposition {
New,
Replay,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn replay_is_safe_but_conflict_is_rejected() {
let mut registry = PlaybackRegistry::default();
let run = RunId::new("run");
let playback = PlaybackId::new("playback");
assert!(matches!(
registry.register_start(run.clone(), 1, playback.clone(), "first".to_owned()),
Ok(StartDisposition::New)
));
assert!(matches!(
registry.register_start(run.clone(), 1, playback.clone(), "first".to_owned()),
Ok(StartDisposition::Replay)
));
assert!(registry.register_start(run, 1, playback, "different".to_owned()).is_err());
}
}
tools/tuliprox-testkit/src/custom_video.rs
+124
View File
@@ -0,0 +1,124 @@
use crate::TestkitError;
use serde::{Deserialize, Serialize};
use std::{collections::BTreeMap, path::Path};
const PREFIX: &[u8] = b"TPX-CVS-";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CustomVideoKind {
ChannelUnavailable,
UserConnectionsExhausted,
ProviderConnectionsExhausted,
LowPriorityPreempted,
UserAccountExpired,
PanelApiProvisioning,
HlsSessionOrLeaseExpired,
}
impl CustomVideoKind {
fn parse(value: &str) -> Option<Self> {
match value {
"CHANNEL_UNAVAILABLE" => Some(Self::ChannelUnavailable),
"USER_CONNECTIONS_EXHAUSTED" => Some(Self::UserConnectionsExhausted),
"PROVIDER_CONNECTIONS_EXHAUSTED" => Some(Self::ProviderConnectionsExhausted),
"LOW_PRIORITY_PREEMPTED" => Some(Self::LowPriorityPreempted),
"USER_ACCOUNT_EXPIRED" => Some(Self::UserAccountExpired),
"PANEL_API_PROVISIONING" => Some(Self::PanelApiProvisioning),
"HLS_SESSION_OR_LEASE_EXPIRED" => Some(Self::HlsSessionOrLeaseExpired),
_ => None,
}
}
}
#[derive(Debug, Default)]
pub struct WireMarkerScanner {
retained: Vec<u8>,
}
impl WireMarkerScanner {
pub fn push(&mut self, bytes: &[u8]) -> Result<Option<CustomVideoKind>, TestkitError> {
self.retained.extend_from_slice(bytes);
if let Some(position) = self.retained.windows(PREFIX.len()).position(|window| window == PREFIX) {
let value_start = position + PREFIX.len();
let Some(value_end) =
self.retained[value_start..].iter().position(|byte| !matches!(byte, b'A'..=b'Z' | b'_'))
else {
self.limit_buffer();
return Ok(None);
};
let marker = std::str::from_utf8(&self.retained[value_start..value_start + value_end])
.map_err(|error| TestkitError::Protocol(format!("invalid custom-video marker: {error}")))?;
let kind = CustomVideoKind::parse(marker)
.ok_or_else(|| TestkitError::Protocol(format!("unknown custom-video marker {marker}")))?;
self.retained.clear();
return Ok(Some(kind));
}
self.limit_buffer();
Ok(None)
}
fn limit_buffer(&mut self) {
const MAX_RETAINED: usize = 128;
if self.retained.len() > MAX_RETAINED {
let start = self.retained.len() - MAX_RETAINED;
self.retained.drain(..start);
}
}
}
#[must_use]
pub fn fixture_digest(bytes: &[u8]) -> String { blake3::hash(bytes).to_hex().to_string() }
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct FixtureManifest {
pub schema_version: u16,
pub fixtures: BTreeMap<String, String>,
}
/// Verifies fixture installation bytes only. It must never be used to classify
/// bytes observed after Tuliprox has processed a transport stream.
pub fn verify_fixture_manifest(root: &Path, manifest: &FixtureManifest) -> Result<(), TestkitError> {
if manifest.schema_version != 1 {
return Err(TestkitError::Configuration("unsupported custom-video fixture manifest schema".to_owned()));
}
for (relative_path, expected_digest) in &manifest.fixtures {
let path = Path::new(relative_path);
if path.is_absolute() || path.components().any(|component| matches!(component, std::path::Component::ParentDir))
{
return Err(TestkitError::Configuration(format!("unsafe fixture path {relative_path}")));
}
let bytes = std::fs::read(root.join(path))?;
let actual_digest = fixture_digest(&bytes);
if actual_digest != *expected_digest {
return Err(TestkitError::Protocol(format!("fixture digest mismatch for {relative_path}")));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn recognizes_marker_split_across_transport_chunks() {
let mut scanner = WireMarkerScanner::default();
assert!(scanner.push(b"ignored TPX-CVS-USER_").unwrap().is_none());
assert_eq!(scanner.push(b"CONNECTIONS_EXHAUSTED;").unwrap(), Some(CustomVideoKind::UserConnectionsExhausted));
}
#[test]
fn fixture_digest_is_deterministic() {
assert_eq!(fixture_digest(b"fixture"), fixture_digest(b"fixture"));
}
#[test]
fn checked_in_fixture_manifest_matches_installation_bytes() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test/fixtures");
let manifest_path = root.join("testkit/custom-video-fixtures.json");
let manifest: FixtureManifest = serde_json::from_slice(&std::fs::read(manifest_path).unwrap()).unwrap();
assert!(verify_fixture_manifest(&root, &manifest).is_ok());
}
}
tools/tuliprox-testkit/src/discovery.rs
+78
View File
@@ -0,0 +1,78 @@
use crate::TestkitError;
use std::collections::HashMap;
/// Maps a stable synthetic-origin marker to the virtual playback URL published by Tuliprox.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VirtualIdMap {
urls: HashMap<u32, String>,
named_urls: HashMap<String, String>,
}
impl VirtualIdMap {
pub fn from_m3u(document: &str) -> Result<Self, TestkitError> {
let mut urls = HashMap::new();
let mut named_urls = HashMap::new();
let mut pending_marker = None;
let mut pending_name = None;
for raw_line in document.lines() {
let line = raw_line.trim();
if line.starts_with("#EXTINF:") {
pending_marker = marker_from_extinf(line);
pending_name = name_from_extinf(line);
continue;
}
if line.is_empty() || line.starts_with('#') {
continue;
}
if let Some(marker) = pending_marker.take() {
if urls.insert(marker, line.to_owned()).is_some() {
return Err(TestkitError::Configuration(format!("duplicate origin marker {marker} in playlist")));
}
}
if let Some(name) = pending_name.take() {
named_urls.insert(name, line.to_owned());
}
}
if urls.is_empty() && named_urls.is_empty() {
return Err(TestkitError::Configuration("playlist has no testkit marker entries".to_owned()));
}
Ok(Self { urls, named_urls })
}
pub fn playback_url(&self, marker: u32) -> Result<&str, TestkitError> {
self.urls
.get(&marker)
.map(String::as_str)
.ok_or_else(|| TestkitError::Configuration(format!("origin marker {marker} was not published by Tuliprox")))
}
pub fn named_playback_url(&self, name: &str) -> Result<&str, TestkitError> {
self.named_urls
.get(name)
.map(String::as_str)
.ok_or_else(|| TestkitError::Configuration(format!("named media {name} was not published by Tuliprox")))
}
}
fn marker_from_extinf(line: &str) -> Option<u32> {
let value = line.split("tvg-id=\"").nth(1)?.split('"').next()?;
value.strip_prefix("test-")?.parse().ok()
}
fn name_from_extinf(line: &str) -> Option<String> {
let value = line.split("tvg-id=\"").nth(1)?.split('"').next()?;
value.strip_prefix("test-vod-").map(str::to_owned)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn discovers_virtual_playback_urls_without_assuming_their_ids() {
let playlist = "#EXTM3U\n#EXTINF:-1 tvg-id=\"test-17\",News\nhttp://tuliprox/m3u/abc/17\n#EXTINF:-1 tvg-id=\"test-vod-movie.mkv\" type=\"movie\",Movie\nhttp://tuliprox/movie/abc/def/1.mkv\n";
let map = VirtualIdMap::from_m3u(playlist).unwrap();
assert_eq!(map.playback_url(17).unwrap(), "http://tuliprox/m3u/abc/17");
assert_eq!(map.named_playback_url("movie.mkv").unwrap(), "http://tuliprox/movie/abc/def/1.mkv");
}
}
tools/tuliprox-testkit/src/faults.rs
+50
View File
@@ -0,0 +1,50 @@
use crate::TestkitError;
use std::collections::HashMap;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OriginFault {
CloseBeforeFirstByte,
CloseAfterBytes(u64),
StallAfterBytes(u64),
CorruptNextFrame,
UnexpectedMarker(u32),
}
#[derive(Debug, Default)]
pub struct FaultSchedule {
faults: HashMap<String, OriginFault>,
}
impl FaultSchedule {
pub fn insert(&mut self, fault_id: String, fault: OriginFault) -> Result<(), TestkitError> {
if self.faults.contains_key(&fault_id) {
return Err(TestkitError::Configuration(format!("fault ID {fault_id} already exists")));
}
self.faults.insert(fault_id, fault);
Ok(())
}
pub fn take(&mut self, fault_id: &str) -> Option<OriginFault> { self.faults.remove(fault_id) }
pub fn clear(&mut self, fault_id: &str) -> Result<(), TestkitError> {
self.faults
.remove(fault_id)
.map(|_| ())
.ok_or_else(|| TestkitError::Configuration(format!("fault ID {fault_id} does not exist")))
}
pub fn clear_all(&mut self) { self.faults.clear(); }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fault_is_consumed_only_once() {
let mut schedule = FaultSchedule::default();
schedule.insert("close".to_owned(), OriginFault::CloseBeforeFirstByte).unwrap();
assert_eq!(schedule.take("close"), Some(OriginFault::CloseBeforeFirstByte));
assert_eq!(schedule.take("close"), None);
}
}
tools/tuliprox-testkit/src/frame.rs
+251
View File
@@ -0,0 +1,251 @@
use crate::{
protocol::{OriginStreamId, RunId},
TestkitError,
};
use bytes::{Buf, BufMut, Bytes, BytesMut};
use crc32fast::hash;
pub const MAGIC: [u8; 8] = *b"TPXTEST\0";
const VERSION: u8 = 1;
const FIXED_HEADER_LEN: usize = 8 + 1 + 1 + 2 + 16 + 16 + 4 + 8 + 8 + 4 + 4 + 4;
const FIXED_HEADER_LEN_U16: u16 = 76;
const TRAILER_LEN: usize = 4;
pub const MAX_PAYLOAD_LEN: usize = 64 * 1024;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Frame {
pub run_id: [u8; 16],
pub origin_stream_id: [u8; 16],
pub channel_marker: u32,
pub sequence: u64,
pub origin_mono_nanos: u64,
pub configured_bps: u32,
pub payload: Bytes,
}
impl Frame {
#[must_use]
pub fn synthetic(run_id: &RunId, stream_id: &OriginStreamId, marker: u32, sequence: u64) -> Self {
let mut run = [0_u8; 16];
let mut stream = [0_u8; 16];
run.copy_from_slice(&blake3::hash(run_id.0.as_bytes()).as_bytes()[..16]);
stream.copy_from_slice(&blake3::hash(stream_id.0.as_bytes()).as_bytes()[..16]);
let payload = format!("marker={marker};sequence={sequence}\n").into_bytes();
Self {
run_id: run,
origin_stream_id: stream,
channel_marker: marker,
sequence,
origin_mono_nanos: 0,
configured_bps: 64_000,
payload: Bytes::from(payload),
}
}
pub fn encode(&self, output: &mut BytesMut) -> Result<(), TestkitError> {
let payload_len = self.payload.len();
if payload_len > MAX_PAYLOAD_LEN {
return Err(TestkitError::Protocol("frame payload exceeds configured limit".to_owned()));
}
let start = output.len();
output.reserve(FIXED_HEADER_LEN + payload_len + TRAILER_LEN);
output.put_slice(&MAGIC);
output.put_u8(VERSION);
output.put_u8(0);
output.put_u16(FIXED_HEADER_LEN_U16);
output.put_slice(&self.run_id);
output.put_slice(&self.origin_stream_id);
output.put_u32(self.channel_marker);
output.put_u64(self.sequence);
output.put_u64(self.origin_mono_nanos);
output.put_u32(self.configured_bps);
output.put_u32(
u32::try_from(payload_len)
.map_err(|_| TestkitError::Protocol("frame payload length cannot be encoded".to_owned()))?,
);
let header_crc = hash(&output[start..]);
output.put_u32(header_crc);
output.put_slice(&self.payload);
output.put_u32(hash(&self.payload));
Ok(())
}
}
#[derive(Debug, Default)]
pub struct FrameDecoder {
buffer: BytesMut,
}
/// Verifies that decoded frames belong to one requested synthetic stream.
/// A late subscriber may start at any sequence, but it can never move backwards
/// or silently switch to another upstream response.
#[derive(Debug)]
pub struct FrameValidator {
expected_run_id: [u8; 16],
expected_marker: u32,
origin_stream_id: Option<[u8; 16]>,
last_sequence: Option<u64>,
}
impl FrameValidator {
#[must_use]
pub fn new(run_id: &RunId, expected_marker: u32) -> Self {
let mut expected_run_id = [0_u8; 16];
expected_run_id.copy_from_slice(&blake3::hash(run_id.0.as_bytes()).as_bytes()[..16]);
Self { expected_run_id, expected_marker, origin_stream_id: None, last_sequence: None }
}
pub fn validate(&mut self, frame: &Frame) -> Result<(), TestkitError> {
if frame.run_id != self.expected_run_id {
return Err(TestkitError::Protocol("received a frame for a foreign test run".to_owned()));
}
if frame.channel_marker != self.expected_marker {
return Err(TestkitError::Protocol(format!(
"marker mismatch: expected {}, got {}",
self.expected_marker, frame.channel_marker
)));
}
if let Some(origin_stream_id) = self.origin_stream_id {
if origin_stream_id != frame.origin_stream_id {
return Err(TestkitError::Protocol("origin stream changed during a playback".to_owned()));
}
} else {
self.origin_stream_id = Some(frame.origin_stream_id);
}
if let Some(previous) = self.last_sequence {
if frame.sequence <= previous {
return Err(TestkitError::Protocol(format!(
"non-increasing frame sequence: previous {previous}, current {}",
frame.sequence
)));
}
}
let expected_payload = format!("marker={};sequence={}\n", frame.channel_marker, frame.sequence);
if frame.payload.as_ref() != expected_payload.as_bytes() {
return Err(TestkitError::Protocol(
"synthetic frame payload does not match marker and sequence".to_owned(),
));
}
self.last_sequence = Some(frame.sequence);
Ok(())
}
}
impl FrameDecoder {
pub fn push(&mut self, bytes: &[u8]) -> Result<Vec<Frame>, TestkitError> {
self.buffer.extend_from_slice(bytes);
let mut frames = Vec::new();
loop {
let Some(position) = self.buffer.windows(MAGIC.len()).position(|window| window == MAGIC) else {
self.keep_magic_suffix();
return Ok(frames);
};
if position != 0 {
self.buffer.advance(position);
}
if self.buffer.len() < FIXED_HEADER_LEN {
return Ok(frames);
}
let header_len = u16::from_be_bytes([self.buffer[10], self.buffer[11]]) as usize;
if header_len != FIXED_HEADER_LEN || self.buffer[8] != VERSION {
self.buffer.advance(1);
continue;
}
let expected_header_crc = u32::from_be_bytes(
self.buffer[FIXED_HEADER_LEN - 4..FIXED_HEADER_LEN]
.try_into()
.map_err(|_| TestkitError::Protocol("invalid header length".to_owned()))?,
);
if hash(&self.buffer[..FIXED_HEADER_LEN - 4]) != expected_header_crc {
self.buffer.advance(1);
continue;
}
let payload_len = u32::from_be_bytes(
self.buffer[68..72]
.try_into()
.map_err(|_| TestkitError::Protocol("invalid payload length".to_owned()))?,
) as usize;
if payload_len > MAX_PAYLOAD_LEN {
self.buffer.advance(1);
continue;
}
let total_len = FIXED_HEADER_LEN + payload_len + TRAILER_LEN;
if self.buffer.len() < total_len {
return Ok(frames);
}
let payload_start = FIXED_HEADER_LEN;
let payload_end = payload_start + payload_len;
let expected_payload_crc = u32::from_be_bytes(
self.buffer[payload_end..total_len]
.try_into()
.map_err(|_| TestkitError::Protocol("invalid trailer length".to_owned()))?,
);
if hash(&self.buffer[payload_start..payload_end]) != expected_payload_crc {
self.buffer.advance(1);
continue;
}
let mut run_id = [0_u8; 16];
let mut origin_stream_id = [0_u8; 16];
run_id.copy_from_slice(&self.buffer[12..28]);
origin_stream_id.copy_from_slice(&self.buffer[28..44]);
let frame = Frame {
run_id,
origin_stream_id,
channel_marker: u32::from_be_bytes(
self.buffer[44..48].try_into().map_err(|_| TestkitError::Protocol("invalid marker".to_owned()))?,
),
sequence: u64::from_be_bytes(
self.buffer[48..56]
.try_into()
.map_err(|_| TestkitError::Protocol("invalid sequence".to_owned()))?,
),
origin_mono_nanos: u64::from_be_bytes(
self.buffer[56..64]
.try_into()
.map_err(|_| TestkitError::Protocol("invalid timestamp".to_owned()))?,
),
configured_bps: u32::from_be_bytes(
self.buffer[64..68].try_into().map_err(|_| TestkitError::Protocol("invalid bitrate".to_owned()))?,
),
payload: self.buffer[payload_start..payload_end].to_vec().into(),
};
self.buffer.advance(total_len);
frames.push(frame);
}
}
fn keep_magic_suffix(&mut self) {
let retained = self.buffer.len().min(MAGIC.len() - 1);
let start = self.buffer.len() - retained;
let suffix = self.buffer.split_off(start);
self.buffer = suffix;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn decodes_every_network_split() {
let frame = Frame::synthetic(&RunId::new("run"), &OriginStreamId::new("origin"), 17, 0);
let mut encoded = BytesMut::new();
frame.encode(&mut encoded).unwrap();
for split in 0..=encoded.len() {
let mut decoder = FrameDecoder::default();
let mut frames = decoder.push(&encoded[..split]).unwrap();
frames.extend(decoder.push(&encoded[split..]).unwrap());
assert_eq!(frames, vec![frame.clone()]);
}
}
#[test]
fn validator_rejects_foreign_runs_and_sequence_replay() {
let run = RunId::new("run");
let stream = OriginStreamId::new("origin");
let mut validator = FrameValidator::new(&run, 17);
assert!(validator.validate(&Frame::synthetic(&run, &stream, 17, 4)).is_ok());
assert!(validator.validate(&Frame::synthetic(&run, &stream, 17, 4)).is_err());
assert!(validator.validate(&Frame::synthetic(&RunId::new("other"), &stream, 17, 5)).is_err());
}
}
tools/tuliprox-testkit/src/hls.rs
+124
View File
@@ -0,0 +1,124 @@
use crate::{
frame::{FrameDecoder, FrameValidator},
protocol::RunId,
TestkitError,
};
use futures::StreamExt;
use std::collections::BTreeMap;
use url::Url;
const SEGMENT_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15);
pub async fn read_segments(
manifest_url: &str,
expected_run_id: &RunId,
expected_marker: u32,
minimum_frames: u64,
headers: &BTreeMap<String, String>,
) -> Result<u64, TestkitError> {
let client = reqwest::Client::builder().http1_only().build()?;
let manifest =
request_with_headers(client.get(manifest_url), headers).send().await?.error_for_status()?.text().await?;
let base = Url::parse(manifest_url).map_err(|error| TestkitError::Configuration(error.to_string()))?;
let segments = manifest.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#'));
let mut frames = 0;
let mut validator = FrameValidator::new(expected_run_id, expected_marker);
for segment in segments {
let segment_url = base.join(segment).map_err(|error| TestkitError::Configuration(error.to_string()))?;
let response = request_with_headers(client.get(segment_url), headers).send().await?.error_for_status()?;
let mut decoder = FrameDecoder::default();
let mut body = response.bytes_stream();
loop {
let chunk = match tokio::time::timeout(SEGMENT_IDLE_TIMEOUT, body.next()).await {
Ok(Some(chunk)) => chunk?,
Ok(None) => break,
Err(_) => return Err(TestkitError::Protocol("HLS segment idle timeout".to_owned())),
};
for frame in decoder.push(&chunk)? {
validator.validate(&frame)?;
frames += 1;
}
}
if frames >= minimum_frames {
return Ok(frames);
}
}
Err(TestkitError::Protocol(format!("HLS produced {frames} valid frames, expected at least {minimum_frames}")))
}
fn request_with_headers(
mut request: reqwest::RequestBuilder,
headers: &BTreeMap<String, String>,
) -> reqwest::RequestBuilder {
for (name, value) in headers {
request = request.header(name, value);
}
request
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{frame::Frame, protocol::OriginStreamId};
use axum::{
body::Body,
extract::OriginalUri,
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
routing::get,
Router,
};
use bytes::BytesMut;
use std::sync::Arc;
use tokio::{net::TcpListener, sync::Mutex};
#[test]
fn manifest_entries_exclude_directives() {
let entries = "#EXTM3U\n#EXTINF:1,\n0.ts\n"
.lines()
.map(str::trim)
.filter(|line| !line.is_empty() && !line.starts_with('#'))
.collect::<Vec<_>>();
assert_eq!(entries, vec!["0.ts"]);
}
#[tokio::test]
async fn forwards_actor_headers_to_manifest_and_segments() -> Result<(), TestkitError> {
let run_id = RunId::new("hls-auth");
let mut payload = BytesMut::new();
Frame::synthetic(&run_id, &OriginStreamId::new("segment"), 17, 0).encode(&mut payload)?;
let payload = Arc::new(payload.freeze());
let requests = Arc::new(Mutex::new(Vec::new()));
let app = Router::new().fallback(get({
let payload = Arc::clone(&payload);
let requests = Arc::clone(&requests);
move |headers: HeaderMap, uri: OriginalUri| {
let payload = Arc::clone(&payload);
let requests = Arc::clone(&requests);
async move {
if headers.get("x-testkit-token").and_then(|value| value.to_str().ok()) != Some("accepted") {
return StatusCode::UNAUTHORIZED.into_response();
}
requests.lock().await.push(uri.path().to_owned());
match uri.path() {
"/index.m3u8" => "#EXTM3U\n#EXTINF:1,\nsegment.ts\n".into_response(),
"/segment.ts" => Response::new(Body::from(payload.as_ref().clone())),
_ => StatusCode::NOT_FOUND.into_response(),
}
}
}
}));
let listener = TcpListener::bind("127.0.0.1:0").await?;
let address = listener.local_addr()?;
let server = tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
let headers = BTreeMap::from([(String::from("x-testkit-token"), String::from("accepted"))]);
let received = read_segments(&format!("http://{address}/index.m3u8"), &run_id, 17, 1, &headers).await?;
server.abort();
assert_eq!(received, 1);
assert_eq!(requests.lock().await.as_slice(), ["/index.m3u8", "/segment.ts"]);
Ok(())
}
}
tools/tuliprox-testkit/src/lib.rs
+57
View File
@@ -0,0 +1,57 @@
#![forbid(unsafe_code)]
pub mod adaptive;
pub mod bootstrap;
pub mod config;
pub mod control;
pub mod custom_video;
pub mod discovery;
pub mod faults;
pub mod frame;
pub mod hls;
pub mod observation;
pub mod oracle;
pub mod origin_events;
pub mod origin_transport;
pub mod policy;
pub mod protocol;
pub mod report;
pub mod scheduler;
pub mod secret;
pub mod shared;
pub mod transport;
pub mod vod;
use std::process::ExitCode;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum TestkitError {
#[error("invalid configuration: {0}")]
Configuration(String),
#[error("protocol error: {0}")]
Protocol(String),
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("HTTP error: {0}")]
Http(#[from] reqwest::Error),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RunExit {
Passed,
Failed,
Invalid,
Inconclusive,
}
impl From<RunExit> for ExitCode {
fn from(value: RunExit) -> Self {
match value {
RunExit::Passed => ExitCode::SUCCESS,
RunExit::Failed => ExitCode::from(1),
RunExit::Invalid => ExitCode::from(2),
RunExit::Inconclusive => ExitCode::from(3),
}
}
}
tools/tuliprox-testkit/src/main.rs
+3107
View File
File diff suppressed because it is too large Load Diff
tools/tuliprox-testkit/src/observation.rs
+49
View File
@@ -0,0 +1,49 @@
use crate::TestkitError;
use serde_json::Value;
#[derive(Debug, Clone)]
pub struct RuntimeSnapshot {
pub status: Value,
pub streams: Value,
pub history: Value,
}
#[derive(Clone)]
pub struct TuliproxObserver {
base_url: String,
client: reqwest::Client,
credentials: Option<(String, String)>,
}
impl TuliproxObserver {
#[must_use]
pub fn new(base_url: impl Into<String>, credentials: Option<(String, String)>) -> Self {
Self { base_url: base_url.into().trim_end_matches('/').to_owned(), client: reqwest::Client::new(), credentials }
}
pub async fn config(&self) -> Result<Value, TestkitError> { self.get_json("config").await }
pub async fn status(&self) -> Result<Value, TestkitError> { self.get_json("status").await }
pub async fn streams(&self) -> Result<Value, TestkitError> { self.get_json("streams").await }
pub async fn stream_history(&self) -> Result<Value, TestkitError> { self.get_json("stream-history").await }
pub async fn runtime_snapshot(&self) -> Result<RuntimeSnapshot, TestkitError> {
let (status, streams) = tokio::join!(self.status(), self.streams());
Ok(RuntimeSnapshot { status: status?, streams: streams?, history: Value::Null })
}
async fn get_json(&self, endpoint: &str) -> Result<Value, TestkitError> {
let request = self.client.get(format!("{}/{}", self.base_url, endpoint));
let request = match &self.credentials {
Some((username, password)) => request.basic_auth(username, Some(password)),
None => request,
};
let response = request.send().await?;
if !response.status().is_success() {
return Err(TestkitError::Protocol(format!("observer endpoint {endpoint} returned {}", response.status())));
}
response.json().await.map_err(TestkitError::Http)
}
}
tools/tuliprox-testkit/src/oracle.rs
+237
View File
@@ -0,0 +1,237 @@
use crate::TestkitError;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AdmissionStrategy {
EvictUserSameIpOldest,
EvictUserSameIpLatest,
EvictUserOldest,
EvictUserLatest,
GraceInstantStream,
GraceHoldStream,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum GraceMode {
Instant,
HoldStream,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Request {
pub playback_id: String,
pub client_ip: String,
pub started_order: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Decision {
Admit,
AdmitSoft,
Deny,
Evict { playback_id: String },
Grace { mode: GraceMode },
}
#[derive(Debug)]
pub struct AdmissionOracle {
max_connections: usize,
soft_connections: usize,
strategies: Vec<AdmissionStrategy>,
grace: Option<GraceMode>,
active: HashMap<String, Request>,
}
impl AdmissionOracle {
#[must_use]
pub fn new(max_connections: usize, strategies: Vec<AdmissionStrategy>) -> Self {
Self { max_connections, soft_connections: 0, strategies, grace: None, active: HashMap::new() }
}
#[must_use]
pub fn with_policy(
max_connections: usize,
soft_connections: usize,
strategies: Vec<AdmissionStrategy>,
grace: Option<GraceMode>,
) -> Self {
Self { max_connections, soft_connections, strategies, grace, active: HashMap::new() }
}
pub fn admit_with_evicted(&mut self, request: Request) -> Result<(Decision, Option<Request>), TestkitError> {
if self.active.contains_key(&request.playback_id) {
return Err(TestkitError::Configuration("duplicate oracle playback".to_owned()));
}
if self.active.len() < self.max_connections {
self.active.insert(request.playback_id.clone(), request);
return Ok((Decision::Admit, None));
}
if self.active.len() < self.max_connections.saturating_add(self.soft_connections) {
self.active.insert(request.playback_id.clone(), request);
return Ok((Decision::AdmitSoft, None));
}
for strategy in &self.strategies {
if let Some(mode) = grace_mode(*strategy) {
return Ok((Decision::Grace { mode }, None));
}
let candidate = self.select(&request, *strategy)?;
if let Some(candidate) = candidate {
let victim = candidate.playback_id.clone();
let evicted_req = self.active.remove(&victim);
self.active.insert(request.playback_id.clone(), request);
return Ok((Decision::Evict { playback_id: victim }, evicted_req));
}
}
Ok((
match self.grace {
Some(mode) => Decision::Grace { mode },
None => Decision::Deny,
},
None,
))
}
pub fn admit(&mut self, request: Request) -> Result<Decision, TestkitError> {
self.admit_with_evicted(request).map(|(decision, _)| decision)
}
pub fn rollback(&mut self, playback_id: &str, evicted: Option<Request>) {
self.active.remove(playback_id);
if let Some(restored) = evicted {
self.active.insert(restored.playback_id.clone(), restored);
}
}
pub fn release(&mut self, playback_id: &str) -> Result<(), TestkitError> {
self.active
.remove(playback_id)
.map(|_| ())
.ok_or_else(|| TestkitError::Configuration(format!("cannot release inactive playback {playback_id}")))
}
fn select(&self, incoming: &Request, strategy: AdmissionStrategy) -> Result<Option<&Request>, TestkitError> {
if grace_mode(strategy).is_some() {
return Ok(None);
}
let same_ip =
matches!(strategy, AdmissionStrategy::EvictUserSameIpOldest | AdmissionStrategy::EvictUserSameIpLatest);
let latest = matches!(strategy, AdmissionStrategy::EvictUserSameIpLatest | AdmissionStrategy::EvictUserLatest);
let candidates = self.active.values().filter(|candidate| !same_ip || candidate.client_ip == incoming.client_ip);
let selected = if latest {
candidates.max_by_key(|candidate| candidate.started_order)
} else {
candidates.min_by_key(|candidate| candidate.started_order)
};
let Some(selected) = selected else { return Ok(None) };
let tied = self.active.values().any(|candidate| {
candidate.playback_id != selected.playback_id
&& (!same_ip || candidate.client_ip == incoming.client_ip)
&& candidate.started_order == selected.started_order
});
if tied {
return Err(TestkitError::Protocol("candidate ages are ambiguous at SUT timestamp resolution".to_owned()));
}
Ok(Some(selected))
}
}
const fn grace_mode(strategy: AdmissionStrategy) -> Option<GraceMode> {
match strategy {
AdmissionStrategy::GraceInstantStream => Some(GraceMode::Instant),
AdmissionStrategy::GraceHoldStream => Some(GraceMode::HoldStream),
AdmissionStrategy::EvictUserSameIpOldest
| AdmissionStrategy::EvictUserSameIpLatest
| AdmissionStrategy::EvictUserOldest
| AdmissionStrategy::EvictUserLatest => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn latest_replaces_latest_candidate() {
let mut oracle = AdmissionOracle::new(2, vec![AdmissionStrategy::EvictUserSameIpLatest]);
for (id, order) in [("a", 1), ("b", 2)] {
assert!(matches!(
oracle.admit(Request {
playback_id: id.to_owned(),
client_ip: "10.0.0.1".to_owned(),
started_order: order
}),
Ok(Decision::Admit)
));
}
assert!(
matches!(oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 3 }), Ok(Decision::Evict { playback_id }) if playback_id == "b")
);
}
#[test]
fn soft_slot_precedes_eviction() {
let mut oracle =
AdmissionOracle::with_policy(1, 1, vec![AdmissionStrategy::EvictUserLatest], Some(GraceMode::HoldStream));
assert!(matches!(
oracle.admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }),
Ok(Decision::Admit)
));
assert!(matches!(
oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }),
Ok(Decision::AdmitSoft)
));
assert!(
matches!(oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.3".to_owned(), started_order: 3 }), Ok(Decision::Evict { playback_id }) if playback_id == "b")
);
oracle.release("a").unwrap();
oracle.release("c").unwrap();
assert!(matches!(
oracle.admit(Request { playback_id: "d".to_owned(), client_ip: "10.0.0.4".to_owned(), started_order: 4 }),
Ok(Decision::Admit)
));
assert!(matches!(
oracle.admit(Request { playback_id: "e".to_owned(), client_ip: "10.0.0.5".to_owned(), started_order: 5 }),
Ok(Decision::AdmitSoft)
));
}
#[test]
fn grace_applies_when_capacity_has_no_eviction_strategy() {
let mut oracle = AdmissionOracle::with_policy(1, 0, Vec::new(), Some(GraceMode::HoldStream));
assert!(oracle
.admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 })
.is_ok());
assert!(matches!(
oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }),
Ok(Decision::Grace { mode: GraceMode::HoldStream })
));
}
#[test]
fn grace_strategy_is_evaluated_in_its_configured_order() {
let mut oracle = AdmissionOracle::new(1, vec![AdmissionStrategy::GraceInstantStream]);
assert!(oracle
.admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 })
.is_ok());
assert!(matches!(
oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }),
Ok(Decision::Grace { mode: GraceMode::Instant })
));
}
#[test]
fn equal_candidate_ages_are_ambiguous() {
let mut oracle = AdmissionOracle::new(2, vec![AdmissionStrategy::EvictUserOldest]);
for id in ["a", "b"] {
assert!(oracle
.admit(Request { playback_id: id.to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 })
.is_ok());
}
assert!(oracle
.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 })
.is_err());
}
}
tools/tuliprox-testkit/src/origin_events.rs
+127
View File
@@ -0,0 +1,127 @@
use serde::{Deserialize, Serialize};
use std::time::{SystemTime, UNIX_EPOCH};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum BodyCloseReason {
Completed,
ClientDisconnected,
Evicted,
Stalled,
Error(String),
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum OriginEventKind {
TcpAccepted {
conn_id: u64,
remote_addr: String,
},
RequestStarted {
conn_id: u64,
request_id: u64,
method: String,
path: String,
range: Option<String>,
user_agent: Option<String>,
account: Option<String>,
},
BodyStarted {
conn_id: u64,
request_id: u64,
content_length: Option<u64>,
},
BodyClosed {
conn_id: u64,
request_id: u64,
bytes_emitted: u64,
duration_ms: u64,
reason: BodyCloseReason,
},
TcpClosed {
conn_id: u64,
duration_ms: u64,
bytes_read: u64,
bytes_written: u64,
reason: String,
},
LimitRejected {
conn_id: u64,
request_id: u64,
limit: usize,
current_active: usize,
},
EvictionTriggered {
evicted_conn_id: u64,
evicted_request_id: u64,
triggering_conn_id: u64,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OriginEvent {
pub seq: u64,
pub timestamp_ns: u128,
pub run_id: String,
#[serde(flatten)]
pub kind: OriginEventKind,
}
impl OriginEvent {
#[must_use]
pub fn new(seq: u64, run_id: impl Into<String>, kind: OriginEventKind) -> Self {
let timestamp_ns = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |duration| duration.as_nanos());
Self { seq, timestamp_ns, run_id: run_id.into(), kind }
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OriginStats {
pub active_tcp_connections: usize,
pub max_active_tcp_connections: usize,
pub active_body_connections: usize,
pub max_active_body_connections: usize,
pub total_tcp_connections: u64,
pub total_requests: u64,
pub total_bytes_emitted: u64,
}
#[must_use]
pub fn redact_path_and_query(raw: &str) -> String {
let Some((path, query)) = raw.split_once('?') else {
return raw.to_owned();
};
let redacted_query: Vec<String> = query
.split('&')
.map(|pair| {
if let Some((key, _)) = pair.split_once('=') {
let lower = key.to_ascii_lowercase();
if lower.contains("pass")
|| lower.contains("token")
|| lower.contains("secret")
|| lower.contains("key")
{
return format!("{key}=<redacted>");
}
}
pair.to_owned()
})
.collect();
format!("{path}?{}", redacted_query.join("&"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redacts_sensitive_query_parameters() {
let path = "/movie/stream.mkv?user=alice&password=secret123&token=abc&run=test-run";
let redacted = redact_path_and_query(path);
assert!(redacted.contains("user=alice"));
assert!(redacted.contains("password=<redacted>"));
assert!(redacted.contains("token=<redacted>"));
assert!(redacted.contains("run=test-run"));
}
}
tools/tuliprox-testkit/src/origin_transport.rs
+484
View File
@@ -0,0 +1,484 @@
use crate::origin_events::{redact_path_and_query, BodyCloseReason, OriginEvent, OriginEventKind, OriginStats};
use pin_project_lite::pin_project;
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
net::SocketAddr,
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
},
task::{Context, Poll},
time::Instant,
};
use tokio::{
io::{AsyncRead, AsyncWrite, ReadBuf},
sync::{oneshot, Mutex},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OriginLimitMode {
ObserveOnly,
RejectNew,
EvictOldest,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OriginPolicy {
pub account_limit: Option<usize>,
pub limit_mode: OriginLimitMode,
}
impl Default for OriginPolicy {
fn default() -> Self { Self { account_limit: None, limit_mode: OriginLimitMode::ObserveOnly } }
}
#[derive(Clone, Debug)]
pub struct OriginConnectionMeta {
pub conn_id: u64,
pub remote_addr: SocketAddr,
}
struct ActiveConnection {
conn_id: u64,
#[allow(dead_code)]
remote_addr: SocketAddr,
connected_at: Instant,
active_body_req_id: Option<u64>,
}
#[derive(Clone)]
pub struct OriginTracker {
run_id: String,
policy: Arc<Mutex<OriginPolicy>>,
events: Arc<Mutex<Vec<OriginEvent>>>,
stats: Arc<Mutex<OriginStats>>,
active_conns: Arc<Mutex<HashMap<u64, ActiveConnection>>>,
eviction_channels: Arc<Mutex<HashMap<u64, oneshot::Sender<()>>>>,
seq_counter: Arc<AtomicU64>,
conn_counter: Arc<AtomicU64>,
req_counter: Arc<AtomicU64>,
}
impl OriginTracker {
#[must_use]
pub fn new(run_id: impl Into<String>, policy: OriginPolicy) -> Self {
Self {
run_id: run_id.into(),
policy: Arc::new(Mutex::new(policy)),
events: Arc::new(Mutex::new(Vec::new())),
stats: Arc::new(Mutex::new(OriginStats::default())),
active_conns: Arc::new(Mutex::new(HashMap::new())),
eviction_channels: Arc::new(Mutex::new(HashMap::new())),
seq_counter: Arc::new(AtomicU64::new(0)),
conn_counter: Arc::new(AtomicU64::new(0)),
req_counter: Arc::new(AtomicU64::new(0)),
}
}
fn next_seq(&self) -> u64 { self.seq_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1) }
pub async fn set_policy(&self, policy: OriginPolicy) { *self.policy.lock().await = policy; }
pub async fn policy(&self) -> OriginPolicy { self.policy.lock().await.clone() }
pub async fn on_tcp_accepted(&self, remote_addr: SocketAddr) -> u64 {
let conn_id = self.conn_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1);
let seq = self.next_seq();
let mut stats = self.stats.lock().await;
stats.active_tcp_connections = stats.active_tcp_connections.saturating_add(1);
stats.total_tcp_connections = stats.total_tcp_connections.saturating_add(1);
if stats.active_tcp_connections > stats.max_active_tcp_connections {
stats.max_active_tcp_connections = stats.active_tcp_connections;
}
drop(stats);
self.active_conns.lock().await.insert(
conn_id,
ActiveConnection { conn_id, remote_addr, connected_at: Instant::now(), active_body_req_id: None },
);
self.events.lock().await.push(OriginEvent::new(
seq,
&self.run_id,
OriginEventKind::TcpAccepted { conn_id, remote_addr: remote_addr.to_string() },
));
conn_id
}
pub async fn on_request_started(
&self,
conn_id: u64,
method: &str,
path_and_query: &str,
range: Option<&str>,
user_agent: Option<&str>,
account: Option<&str>,
) -> Result<u64, (u64, u16)> {
let req_id = self.req_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1);
let seq = self.next_seq();
let mut stats = self.stats.lock().await;
stats.total_requests = stats.total_requests.saturating_add(1);
let active_bodies = stats.active_body_connections;
drop(stats);
let policy = self.policy.lock().await.clone();
if let Some(limit) = policy.account_limit {
if active_bodies >= limit {
match policy.limit_mode {
OriginLimitMode::ObserveOnly => {}
OriginLimitMode::RejectNew => {
let reject_seq = self.next_seq();
self.events.lock().await.push(OriginEvent::new(
reject_seq,
&self.run_id,
OriginEventKind::LimitRejected {
conn_id,
request_id: req_id,
limit,
current_active: active_bodies,
},
));
return Err((req_id, 429));
}
OriginLimitMode::EvictOldest => {
self.evict_oldest_body(conn_id).await;
}
}
}
}
self.events.lock().await.push(OriginEvent::new(
seq,
&self.run_id,
OriginEventKind::RequestStarted {
conn_id,
request_id: req_id,
method: method.to_owned(),
path: redact_path_and_query(path_and_query),
range: range.map(str::to_owned),
user_agent: user_agent.map(str::to_owned),
account: account.map(str::to_owned),
},
));
Ok(req_id)
}
async fn evict_oldest_body(&self, triggering_conn_id: u64) {
let conns = self.active_conns.lock().await;
let oldest_active = conns
.values()
.filter(|c| c.active_body_req_id.is_some() && c.conn_id != triggering_conn_id)
.min_by_key(|c| c.connected_at);
if let Some(oldest) = oldest_active {
let evicted_conn_id = oldest.conn_id;
let evicted_req_id = oldest.active_body_req_id.unwrap_or(0);
drop(conns);
if let Some(sender) = self.eviction_channels.lock().await.remove(&evicted_conn_id) {
let _ = sender.send(());
}
let seq = self.next_seq();
self.events.lock().await.push(OriginEvent::new(
seq,
&self.run_id,
OriginEventKind::EvictionTriggered {
evicted_conn_id,
evicted_request_id: evicted_req_id,
triggering_conn_id,
},
));
}
}
pub async fn on_body_started(
&self,
conn_id: u64,
req_id: u64,
content_length: Option<u64>,
) -> oneshot::Receiver<()> {
let seq = self.next_seq();
let mut stats = self.stats.lock().await;
stats.active_body_connections = stats.active_body_connections.saturating_add(1);
if stats.active_body_connections > stats.max_active_body_connections {
stats.max_active_body_connections = stats.active_body_connections;
}
drop(stats);
if let Some(conn) = self.active_conns.lock().await.get_mut(&conn_id) {
conn.active_body_req_id = Some(req_id);
}
let (tx, rx) = oneshot::channel();
self.eviction_channels.lock().await.insert(conn_id, tx);
self.events.lock().await.push(OriginEvent::new(
seq,
&self.run_id,
OriginEventKind::BodyStarted { conn_id, request_id: req_id, content_length },
));
rx
}
pub async fn on_body_closed(
&self,
conn_id: u64,
req_id: u64,
bytes_emitted: u64,
duration_ms: u64,
reason: BodyCloseReason,
) {
let seq = self.next_seq();
let mut stats = self.stats.lock().await;
stats.active_body_connections = stats.active_body_connections.saturating_sub(1);
stats.total_bytes_emitted = stats.total_bytes_emitted.saturating_add(bytes_emitted);
drop(stats);
self.eviction_channels.lock().await.remove(&conn_id);
if let Some(conn) = self.active_conns.lock().await.get_mut(&conn_id) {
if conn.active_body_req_id == Some(req_id) {
conn.active_body_req_id = None;
}
}
self.events.lock().await.push(OriginEvent::new(
seq,
&self.run_id,
OriginEventKind::BodyClosed { conn_id, request_id: req_id, bytes_emitted, duration_ms, reason },
));
}
pub async fn on_tcp_closed(
&self,
conn_id: u64,
duration_ms: u64,
bytes_read: u64,
bytes_written: u64,
reason: &str,
) {
let seq = self.next_seq();
let mut stats = self.stats.lock().await;
stats.active_tcp_connections = stats.active_tcp_connections.saturating_sub(1);
drop(stats);
self.active_conns.lock().await.remove(&conn_id);
self.eviction_channels.lock().await.remove(&conn_id);
self.events.lock().await.push(OriginEvent::new(
seq,
&self.run_id,
OriginEventKind::TcpClosed { conn_id, duration_ms, bytes_read, bytes_written, reason: reason.to_owned() },
));
}
pub async fn stats(&self) -> OriginStats { self.stats.lock().await.clone() }
pub async fn events(&self) -> Vec<OriginEvent> { self.events.lock().await.clone() }
pub async fn reset(&self) {
*self.stats.lock().await = OriginStats::default();
self.events.lock().await.clear();
self.active_conns.lock().await.clear();
self.eviction_channels.lock().await.clear();
self.seq_counter.store(0, Ordering::Relaxed);
self.conn_counter.store(0, Ordering::Relaxed);
self.req_counter.store(0, Ordering::Relaxed);
}
}
struct TcpDropGuard {
conn_id: u64,
tracker: OriginTracker,
start_time: Instant,
bytes_read: Arc<AtomicU64>,
bytes_written: Arc<AtomicU64>,
closed: Arc<AtomicBool>,
}
impl Drop for TcpDropGuard {
fn drop(&mut self) {
if !self.closed.swap(true, Ordering::SeqCst) {
let conn_id = self.conn_id;
let tracker = self.tracker.clone();
let duration_ms = u64::try_from(self.start_time.elapsed().as_millis()).unwrap_or(u64::MAX);
let bytes_read = self.bytes_read.load(Ordering::Relaxed);
let bytes_written = self.bytes_written.load(Ordering::Relaxed);
tokio::spawn(async move {
tracker.on_tcp_closed(conn_id, duration_ms, bytes_read, bytes_written, "io_dropped").await;
});
}
}
}
pin_project! {
pub struct TrackedIo<T> {
#[pin]
inner: T,
bytes_read: Arc<AtomicU64>,
bytes_written: Arc<AtomicU64>,
_guard: TcpDropGuard,
}
}
impl<T> TrackedIo<T> {
pub fn new(inner: T, conn_id: u64, tracker: OriginTracker) -> (Self, Arc<AtomicBool>) {
let closed = Arc::new(AtomicBool::new(false));
let bytes_read = Arc::new(AtomicU64::new(0));
let bytes_written = Arc::new(AtomicU64::new(0));
let guard = TcpDropGuard {
conn_id,
tracker,
start_time: Instant::now(),
bytes_read: bytes_read.clone(),
bytes_written: bytes_written.clone(),
closed: closed.clone(),
};
(Self { inner, bytes_read, bytes_written, _guard: guard }, closed)
}
}
impl<T: AsyncRead> AsyncRead for TrackedIo<T> {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
let this = self.project();
let prev_len = buf.filled().len();
let poll_res = this.inner.poll_read(cx, buf);
if let Poll::Ready(Ok(())) = &poll_res {
let newly_read = buf.filled().len().saturating_sub(prev_len) as u64;
this.bytes_read.fetch_add(newly_read, Ordering::Relaxed);
}
poll_res
}
}
impl<T: AsyncWrite> AsyncWrite for TrackedIo<T> {
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
let this = self.project();
let poll_res = this.inner.poll_write(cx, buf);
if let Poll::Ready(Ok(written)) = &poll_res {
this.bytes_written.fetch_add(*written as u64, Ordering::Relaxed);
}
poll_res
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
self.project().inner.poll_flush(cx)
}
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
self.project().inner.poll_shutdown(cx)
}
}
pub async fn serve_tracked_media_listener(
listener: tokio::net::TcpListener,
app: axum::Router,
tracker: OriginTracker,
) -> Result<(), std::io::Error> {
loop {
let (tcp_stream, remote_addr) = listener.accept().await?;
let conn_id = tracker.on_tcp_accepted(remote_addr).await;
let (tracked_io, _closed) = TrackedIo::new(tcp_stream, conn_id, tracker.clone());
let io = hyper_util::rt::TokioIo::new(tracked_io);
let app = app.clone();
let conn_meta = OriginConnectionMeta { conn_id, remote_addr };
tokio::spawn(async move {
let service = hyper::service::service_fn(move |req: hyper::Request<hyper::body::Incoming>| {
let mut req = req.map(axum::body::Body::new);
req.extensions_mut().insert(conn_meta.clone());
let mut app = app.clone();
async move { tower_service::Service::call(&mut app, req).await }
});
let builder = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new());
let _ = builder.serve_connection(io, service).await;
});
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{IpAddr, Ipv4Addr};
#[tokio::test]
async fn tracking_records_tcp_lifecycle_and_peak_counters() {
let tracker = OriginTracker::new("test-run", OriginPolicy::default());
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080);
let conn_id = tracker.on_tcp_accepted(addr).await;
assert_eq!(conn_id, 1);
let stats = tracker.stats().await;
assert_eq!(stats.active_tcp_connections, 1);
assert_eq!(stats.max_active_tcp_connections, 1);
let req_id = tracker
.on_request_started(conn_id, "GET", "/live/17.ts", None, Some("test/1"), None)
.await
.expect("request admitted");
assert_eq!(req_id, 1);
let _evict_rx = tracker.on_body_started(conn_id, req_id, None).await;
let stats = tracker.stats().await;
assert_eq!(stats.active_body_connections, 1);
assert_eq!(stats.max_active_body_connections, 1);
tracker.on_body_closed(conn_id, req_id, 1024, 50, BodyCloseReason::Completed).await;
let stats = tracker.stats().await;
assert_eq!(stats.active_body_connections, 0);
assert_eq!(stats.max_active_body_connections, 1);
tracker.on_tcp_closed(conn_id, 100, 200, 1024, "clean_close").await;
let stats = tracker.stats().await;
assert_eq!(stats.active_tcp_connections, 0);
assert_eq!(stats.max_active_tcp_connections, 1);
let events = tracker.events().await;
assert_eq!(events.len(), 5);
}
#[tokio::test]
async fn policy_rejects_new_when_account_limit_reached() {
let policy = OriginPolicy { account_limit: Some(1), limit_mode: OriginLimitMode::RejectNew };
let tracker = OriginTracker::new("test-run", policy);
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080);
let conn1 = tracker.on_tcp_accepted(addr).await;
let req1 = tracker.on_request_started(conn1, "GET", "/live/1.ts", None, None, None).await.unwrap();
let _rx1 = tracker.on_body_started(conn1, req1, None).await;
let conn2 = tracker.on_tcp_accepted(addr).await;
let req2_res = tracker.on_request_started(conn2, "GET", "/live/2.ts", None, None, None).await;
assert_eq!(req2_res, Err((2, 429)));
}
#[tokio::test]
async fn policy_evicts_oldest_when_account_limit_reached() {
let policy = OriginPolicy { account_limit: Some(1), limit_mode: OriginLimitMode::EvictOldest };
let tracker = OriginTracker::new("test-run", policy);
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080);
let conn1 = tracker.on_tcp_accepted(addr).await;
let req1 = tracker.on_request_started(conn1, "GET", "/live/1.ts", None, None, None).await.unwrap();
let mut rx1 = tracker.on_body_started(conn1, req1, None).await;
let conn2 = tracker.on_tcp_accepted(addr).await;
let req2 = tracker.on_request_started(conn2, "GET", "/live/2.ts", None, None, None).await.unwrap();
assert_eq!(req2, 2);
// conn1 should have received eviction signal
assert!(rx1.try_recv().is_ok());
}
}
tools/tuliprox-testkit/src/policy.rs
+265
View File
@@ -0,0 +1,265 @@
use crate::{
config::{GraceContract, PolicyContract},
oracle::{AdmissionStrategy, GraceMode},
TestkitError,
};
use serde_json::Value;
use std::collections::HashMap;
/// Validate the persisted fixture configuration before any playback is started.
///
/// The v1 configuration endpoint intentionally returns configuration files, rather
/// than the atomically loaded runtime configuration. The controller therefore
/// uses this only for isolated fixtures it started itself; a remote instance needs
/// runtime evidence and is reported as inconclusive by the caller.
pub fn validate_fixture_policy(contract: &PolicyContract, config: &Value) -> Result<(), TestkitError> {
let configured_access_control = required_bool(config, "/config/user_access_control")?;
if configured_access_control != contract.user_access_control {
return mismatch("user_access_control", contract.user_access_control, configured_access_control);
}
let stream = required_object(config, "/config/reverse_proxy/stream")?;
let configured_strategies = stream
.get("admission_strategies")
.map(|strategies| {
serde_json::from_value::<Vec<AdmissionStrategy>>(strategies.clone())
.map_err(|error| TestkitError::Protocol(format!("invalid fixture admission_strategies: {error}")))
})
.transpose()?;
if configured_strategies != contract.admission_strategies {
return Err(TestkitError::Configuration(format!(
"fixture admission_strategies mismatch: scenario {:?}, fixture {:?}",
contract.admission_strategies, configured_strategies
)));
}
if let Some(grace) = &contract.grace {
validate_grace(grace, stream, &contract.effective_admission_strategies())?;
}
let configured_users = index_users(config)?;
for (username, expected) in &contract.users {
let configured = configured_users
.get(username.as_str())
.ok_or_else(|| TestkitError::Configuration(format!("fixture policy does not contain user {username}")))?;
let max_connections = usize::try_from(required_u64(configured, "max_connections")?).map_err(|_| {
TestkitError::Protocol("fixture max_connections cannot be represented on this platform".to_owned())
})?;
let soft_connections = usize::try_from(required_u64(configured, "soft_connections")?).map_err(|_| {
TestkitError::Protocol("fixture soft_connections cannot be represented on this platform".to_owned())
})?;
if max_connections != expected.max_connections || soft_connections != expected.soft_connections {
return Err(TestkitError::Configuration(format!(
"fixture policy mismatch for user {username}: scenario max/soft {}/{}, fixture {}/{}",
expected.max_connections, expected.soft_connections, max_connections, soft_connections
)));
}
}
if let Some(expected_limit) = contract.provider_max_connections {
validate_provider_limit(expected_limit.into(), config)?;
}
Ok(())
}
fn validate_provider_limit(expected_limit: usize, config: &Value) -> Result<(), TestkitError> {
let inputs = config
.pointer("/sources/inputs")
.and_then(Value::as_array)
.ok_or_else(|| TestkitError::Protocol("fixture configuration omits sources.inputs".to_owned()))?;
let matching_input = inputs
.iter()
.find(|input| {
input.get("name").and_then(Value::as_str) == Some("testkit-origin")
|| input.get("max_connections").is_some()
})
.ok_or_else(|| {
TestkitError::Configuration(
"fixture sources do not contain a provider input with max_connections".to_owned(),
)
})?;
let configured_limit = matching_input
.get("max_connections")
.and_then(Value::as_u64)
.ok_or_else(|| TestkitError::Configuration("fixture provider input omits max_connections".to_owned()))?;
let configured_limit = usize::try_from(configured_limit).map_err(|_| {
TestkitError::Protocol("fixture provider max_connections cannot be represented on this platform".to_owned())
})?;
if configured_limit != expected_limit {
return Err(TestkitError::Configuration(format!(
"fixture provider max_connections mismatch: scenario {expected_limit}, fixture {configured_limit}"
)));
}
Ok(())
}
fn validate_grace(
expected: &GraceContract,
stream: &serde_json::Map<String, Value>,
strategies: &[AdmissionStrategy],
) -> Result<(), TestkitError> {
let timeout = required_u64_object(stream, "grace_period_millis")?;
if timeout != expected.timeout_millis {
return Err(TestkitError::Configuration(format!(
"fixture grace_period_millis mismatch: scenario {}, fixture {timeout}",
expected.timeout_millis
)));
}
let hold = required_bool_object(stream, "grace_period_hold_stream")?;
let expected_hold = expected.mode == GraceMode::HoldStream;
if hold != expected_hold {
return mismatch("grace_period_hold_stream", expected_hold, hold);
}
// A grace mode without its corresponding strategy would be silently ignored
// by the SUT after capacity is exhausted.
let grace_strategy = match expected.mode {
GraceMode::Instant => "grace_instant_stream",
GraceMode::HoldStream => "grace_hold_stream",
};
let encoded = serde_json::to_value(strategies)
.map_err(|error| TestkitError::Protocol(format!("cannot encode admission strategies: {error}")))?;
if !encoded.as_array().is_some_and(|values| values.iter().any(|value| value == grace_strategy)) {
return Err(TestkitError::Configuration(format!(
"fixture policy grace mode requires {grace_strategy} admission strategy"
)));
}
Ok(())
}
fn index_users(config: &Value) -> Result<HashMap<&str, &Value>, TestkitError> {
let users = config
.pointer("/api_proxy/user")
.and_then(Value::as_array)
.ok_or_else(|| TestkitError::Protocol("fixture configuration omits api_proxy.user".to_owned()))?;
let mut indexed = HashMap::new();
for target in users {
let credentials = target
.get("credentials")
.and_then(Value::as_array)
.ok_or_else(|| TestkitError::Protocol("fixture api_proxy user entry omits credentials".to_owned()))?;
for user in credentials {
let username = user
.get("username")
.and_then(Value::as_str)
.ok_or_else(|| TestkitError::Protocol("fixture user omits username".to_owned()))?;
if indexed.insert(username, user).is_some() {
return Err(TestkitError::Configuration(format!("fixture has duplicate username {username}")));
}
}
}
Ok(indexed)
}
fn required_object<'a>(value: &'a Value, pointer: &str) -> Result<&'a serde_json::Map<String, Value>, TestkitError> {
value
.pointer(pointer)
.and_then(Value::as_object)
.ok_or_else(|| TestkitError::Protocol(format!("fixture configuration omits object {pointer}")))
}
fn required_bool(value: &Value, pointer: &str) -> Result<bool, TestkitError> {
value
.pointer(pointer)
.and_then(Value::as_bool)
.ok_or_else(|| TestkitError::Protocol(format!("fixture configuration omits boolean {pointer}")))
}
fn required_bool_object(value: &serde_json::Map<String, Value>, key: &str) -> Result<bool, TestkitError> {
value.get(key).and_then(Value::as_bool).ok_or_else(|| {
TestkitError::Protocol(format!("fixture configuration omits boolean config.reverse_proxy.stream.{key}"))
})
}
fn required_u64(value: &Value, key: &str) -> Result<u64, TestkitError> {
value
.get(key)
.and_then(Value::as_u64)
.ok_or_else(|| TestkitError::Protocol(format!("fixture user omits integer {key}")))
}
fn required_u64_object(value: &serde_json::Map<String, Value>, key: &str) -> Result<u64, TestkitError> {
value.get(key).and_then(Value::as_u64).ok_or_else(|| {
TestkitError::Protocol(format!("fixture configuration omits integer config.reverse_proxy.stream.{key}"))
})
}
fn mismatch<T: std::fmt::Display>(field: &str, expected: T, actual: T) -> Result<(), TestkitError> {
Err(TestkitError::Configuration(format!("fixture {field} mismatch: scenario {expected}, fixture {actual}")))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::UserPolicy;
use serde_json::json;
fn contract() -> PolicyContract {
PolicyContract {
user_access_control: true,
users: HashMap::from([("alice".to_owned(), UserPolicy { max_connections: 2, soft_connections: 1 })]),
admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpLatest]),
grace: None,
provider_max_connections: None,
expected_provider_slots: None,
}
}
fn fixture() -> Value {
json!({
"config": {"user_access_control": true, "reverse_proxy": {"stream": {"admission_strategies": ["evict_user_same_ip_latest"]}}},
"api_proxy": {"user": [{"credentials": [{"username": "alice", "max_connections": 2, "soft_connections": 1}]}]}
})
}
#[test]
fn accepts_matching_fixture_policy() {
assert!(validate_fixture_policy(&contract(), &fixture()).is_ok());
}
#[test]
fn rejects_mismatched_user_limit() {
let mut config = fixture();
config["api_proxy"]["user"][0]["credentials"][0]["max_connections"] = json!(3);
assert!(validate_fixture_policy(&contract(), &config).is_err());
}
#[test]
fn distinguishes_omitted_and_explicit_admission_strategies() {
let mut config = fixture();
config["config"]["reverse_proxy"]["stream"].as_object_mut().unwrap().remove("admission_strategies");
assert!(matches!(validate_fixture_policy(&contract(), &config), Err(TestkitError::Configuration(_))));
let mut omitted_contract = contract();
omitted_contract.admission_strategies = None;
assert!(validate_fixture_policy(&omitted_contract, &config).is_ok());
}
#[test]
fn validates_provider_max_connections() {
let mut contract_with_provider = contract();
contract_with_provider.provider_max_connections = Some(2);
let mut config_matching = fixture();
config_matching["sources"] = json!({
"inputs": [{"name": "testkit-origin", "max_connections": 2}]
});
assert!(validate_fixture_policy(&contract_with_provider, &config_matching).is_ok());
let mut config_mismatched = fixture();
config_mismatched["sources"] = json!({
"inputs": [{"name": "testkit-origin", "max_connections": 1}]
});
assert!(validate_fixture_policy(&contract_with_provider, &config_mismatched).is_err());
let mut config_missing = fixture();
config_missing["sources"] = json!({
"inputs": [{"name": "other-origin"}]
});
assert!(validate_fixture_policy(&contract_with_provider, &config_missing).is_err());
}
}
tools/tuliprox-testkit/src/protocol.rs
+235
View File
@@ -0,0 +1,235 @@
use crate::TestkitError;
use serde::{Deserialize, Serialize};
use std::time::Duration;
pub const SCHEMA_VERSION: u16 = 1;
macro_rules! id_type {
($name:ident) => {
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct $name(pub String);
impl $name {
#[must_use]
pub fn new(value: impl Into<String>) -> Self { Self(value.into()) }
}
};
}
id_type!(RunId);
id_type!(AgentId);
id_type!(ActorId);
id_type!(PlaybackId);
id_type!(CommandId);
id_type!(OriginStreamId);
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Envelope<T> {
pub schema_version: u16,
pub run_id: RunId,
pub run_generation: u64,
pub message_id: String,
pub agent_id: AgentId,
pub agent_boot_id: String,
pub source_sequence: u64,
pub caused_by_command_id: Option<CommandId>,
pub local_elapsed_nanos: u64,
pub payload: T,
}
impl<T> Envelope<T> {
pub fn validate(&self) -> Result<(), TestkitError> {
if self.schema_version != SCHEMA_VERSION {
return Err(TestkitError::Protocol(format!("unsupported schema version {}", self.schema_version)));
}
if self.run_id.0.is_empty() || self.message_id.is_empty() || self.agent_id.0.is_empty() {
return Err(TestkitError::Protocol("empty envelope identity".to_owned()));
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum Command {
ConfigureRun {
lease_millis: u64,
},
StartPlayback {
playback_id: PlaybackId,
url: String,
#[serde(default)]
headers: std::collections::BTreeMap<String, String>,
#[serde(default)]
expected_run_id: Option<RunId>,
#[serde(default)]
expected_marker: Option<u32>,
#[serde(default)]
required_frames: Option<u64>,
},
StopPlayback {
playback_id: PlaybackId,
},
StopAll,
RenewRunLease {
lease_millis: u64,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RejectionReason {
CustomVideo(crate::custom_video::CustomVideoKind),
HttpStatus(u16),
ClosedBeforeFirstByte,
Other(String),
}
impl std::fmt::Display for RejectionReason {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::CustomVideo(kind) => write!(f, "custom-video({kind:?})"),
Self::HttpStatus(status) => write!(f, "http-status({status})"),
Self::ClosedBeforeFirstByte => write!(f, "closed-before-first-byte"),
Self::Other(msg) => write!(f, "{msg}"),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "outcome", rename_all = "snake_case")]
pub enum PlaybackOutcome {
Streaming { frames: u64, bytes: u64 },
AdmissionRejected { reason: RejectionReason },
HttpError { status: u16 },
TransportError { message: String },
InvalidData { message: String },
UnexpectedEof { frames: u64, bytes: u64 },
IdleTimeout,
ExplicitStop,
InfrastructureError { message: String },
}
impl PlaybackOutcome {
#[must_use]
pub fn is_streaming(&self) -> bool { matches!(self, Self::Streaming { .. }) }
#[must_use]
pub fn is_rejected(&self) -> bool { matches!(self, Self::AdmissionRejected { .. }) }
}
impl std::fmt::Display for PlaybackOutcome {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Streaming { frames, bytes } => write!(f, "streaming({frames} frames, {bytes} bytes)"),
Self::AdmissionRejected { reason } => write!(f, "rejected({reason})"),
Self::HttpError { status } => write!(f, "http_error({status})"),
Self::TransportError { message } => write!(f, "transport_error({message})"),
Self::InvalidData { message } => write!(f, "invalid_data({message})"),
Self::UnexpectedEof { frames, bytes } => write!(f, "unexpected_eof({frames} frames, {bytes} bytes)"),
Self::IdleTimeout => write!(f, "idle_timeout"),
Self::ExplicitStop => write!(f, "explicit_stop"),
Self::InfrastructureError { message } => write!(f, "infrastructure_error({message})"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum PlaybackEvent {
CommandAccepted {
command_id: CommandId,
},
HeadersReceived {
playback_id: PlaybackId,
status: u16,
},
FirstValidFrame {
playback_id: PlaybackId,
sequence: u64,
},
Progress {
playback_id: PlaybackId,
frames: u64,
bytes: u64,
},
Terminal {
playback_id: PlaybackId,
outcome: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
typed_outcome: Option<PlaybackOutcome>,
},
LeaseExpired,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum AgentMessage {
Hello { hostname: String, supported_protocols: Vec<String>, maximum_listeners: u32 },
Ready,
Event { event: PlaybackEvent },
Heartbeat,
}
#[derive(Debug, Clone)]
pub struct RunLease {
generation: u64,
expires_at: tokio::time::Instant,
}
impl RunLease {
#[must_use]
pub fn new(generation: u64, duration: Duration) -> Self {
Self { generation, expires_at: tokio::time::Instant::now() + duration }
}
pub fn renew(&mut self, generation: u64, duration: Duration) -> Result<(), TestkitError> {
if generation != self.generation {
return Err(TestkitError::Protocol("stale run generation".to_owned()));
}
self.expires_at = tokio::time::Instant::now() + duration;
Ok(())
}
#[must_use]
pub fn expired(&self) -> bool { tokio::time::Instant::now() >= self.expires_at }
#[must_use]
pub const fn deadline(&self) -> tokio::time::Instant { self.expires_at }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rejects_unknown_schema() {
let envelope = Envelope {
schema_version: 2,
run_id: RunId::new("run"),
run_generation: 1,
message_id: "message".to_owned(),
agent_id: AgentId::new("agent"),
agent_boot_id: "boot".to_owned(),
source_sequence: 0,
caused_by_command_id: None,
local_elapsed_nanos: 0,
payload: Command::StopAll,
};
assert!(envelope.validate().is_err());
}
#[tokio::test(start_paused = true)]
async fn lease_expiry_and_renewal_use_monotonic_time() {
let mut lease = RunLease::new(1, Duration::from_secs(5));
tokio::time::advance(Duration::from_secs(4)).await;
assert!(!lease.expired());
lease.renew(1, Duration::from_secs(5)).unwrap();
tokio::time::advance(Duration::from_secs(4)).await;
assert!(!lease.expired());
tokio::time::advance(Duration::from_secs(1)).await;
assert!(lease.expired());
}
}
tools/tuliprox-testkit/src/report.rs
+146
View File
@@ -0,0 +1,146 @@
use crate::TestkitError;
use serde::Serialize;
use std::{fs, path::Path};
#[derive(Debug, Serialize, serde::Deserialize)]
pub struct RunReport<'a> {
pub schema_version: u16,
pub scenario: &'a str,
pub outcome: &'a str,
pub policy_hash: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
pub run_id: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error_kind: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error_message: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
pub exit_code: Option<i32>,
pub events: Vec<ReportEvent<'a>>,
}
#[derive(Debug, Serialize, serde::Deserialize)]
pub struct ReportEvent<'a> {
pub source: &'a str,
pub playback_id: &'a str,
pub event: &'a str,
pub detail: &'a str,
}
impl RunReport<'_> {
pub fn write_to(&self, directory: &Path) -> Result<(), TestkitError> {
fs::create_dir_all(directory)?;
let json = serde_json::to_vec_pretty(self)
.map_err(|error| TestkitError::Protocol(format!("cannot serialize report: {error}")))?;
fs::write(directory.join("run.json"), json)?;
fs::write(directory.join("events.ndjson"), self.events_ndjson()?)?;
let run_id_line = self.run_id.map_or_else(String::new, |id| format!("run_id: {id}\n"));
let summary = format!(
"scenario: {}\noutcome: {}\npolicy: {}\n{}events: {}\n",
self.scenario,
self.outcome,
self.policy_hash,
run_id_line,
self.events.len()
);
fs::write(directory.join("summary.txt"), summary)?;
fs::write(directory.join("junit.xml"), self.junit_xml())?;
Ok(())
}
fn junit_xml(&self) -> String {
let failed = usize::from(self.outcome != "passed");
let name = xml_escape(self.scenario);
if failed == 0 {
format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testsuite name=\"tuliprox-testkit\" tests=\"1\" failures=\"0\"><testcase name=\"{name}\"/></testsuite>\n"
)
} else {
format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testsuite name=\"tuliprox-testkit\" tests=\"1\" failures=\"{failed}\"><testcase name=\"{name}\"><failure message=\"{}\"/></testcase></testsuite>\n",
xml_escape(self.outcome)
)
}
}
fn events_ndjson(&self) -> Result<String, TestkitError> {
let mut lines = String::new();
for event in &self.events {
let encoded = serde_json::to_string(event)
.map_err(|error| TestkitError::Protocol(format!("cannot serialize report event: {error}")))?;
lines.push_str(&encoded);
lines.push('\n');
}
Ok(lines)
}
}
fn xml_escape(value: &str) -> String {
value.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;").replace('"', "&quot;").replace('\'', "&apos;")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn junit_marks_failed_runs() {
let report = RunReport {
schema_version: 1,
scenario: "a&b",
outcome: "failed",
policy_hash: "hash",
run_id: None,
error_kind: None,
error_message: None,
exit_code: None,
events: Vec::new(),
};
assert!(report.junit_xml().contains("failures=\"1\""));
assert!(report.junit_xml().contains("a&amp;b"));
}
#[test]
fn events_are_emitted_as_one_json_object_per_line() {
let report = RunReport {
schema_version: 1,
scenario: "scenario",
outcome: "passed",
policy_hash: "hash",
run_id: None,
error_kind: None,
error_message: None,
exit_code: None,
events: vec![ReportEvent {
source: "controller",
playback_id: "playback-a",
event: "started",
detail: "ok",
}],
};
let lines = report.events_ndjson().unwrap();
assert_eq!(lines.lines().count(), 1);
assert_eq!(serde_json::from_str::<serde_json::Value>(lines.trim()).unwrap()["playback_id"], "playback-a");
}
#[test]
fn report_serializes_and_deserializes_with_run_id_and_error() {
let report = RunReport {
schema_version: 1,
scenario: "scenario_fail",
outcome: "failed",
policy_hash: "pre-start",
run_id: Some("test-run-123"),
error_kind: Some("bootstrap_error"),
error_message: Some("address already in use"),
exit_code: Some(4),
events: Vec::new(),
};
let json = serde_json::to_string(&report).unwrap();
let value: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(value["run_id"], "test-run-123");
assert_eq!(value["error_kind"], "bootstrap_error");
assert_eq!(value["error_message"], "address already in use");
assert_eq!(value["exit_code"], 4);
}
}
tools/tuliprox-testkit/src/scheduler.rs
+85
View File
@@ -0,0 +1,85 @@
use crate::TestkitError;
use std::{collections::HashMap, time::Duration};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct EventKey {
pub command_id: String,
pub event: String,
}
#[derive(Debug, Default)]
pub struct Timeline {
anchors: HashMap<EventKey, tokio::time::Instant>,
}
impl Timeline {
pub fn record(&mut self, key: EventKey) -> Result<(), TestkitError> {
if self.anchors.contains_key(&key) {
return Err(TestkitError::Protocol(format!("duplicate causal event {}:{}", key.command_id, key.event)));
}
self.anchors.insert(key, tokio::time::Instant::now());
Ok(())
}
pub fn deadline(&self, key: &EventKey, within: Duration) -> Result<tokio::time::Instant, TestkitError> {
self.anchors.get(key).copied().map(|anchor| anchor + within).ok_or_else(|| {
TestkitError::Configuration(format!("assertion anchor {}:{} was never armed", key.command_id, key.event))
})
}
pub fn require_before(
&self,
key: &EventKey,
within: Duration,
observed_at: tokio::time::Instant,
) -> Result<(), TestkitError> {
if observed_at > self.deadline(key, within)? {
return Err(TestkitError::Protocol(format!(
"event exceeded deadline anchored at {}:{}",
key.command_id, key.event
)));
}
Ok(())
}
}
#[derive(Debug)]
pub struct ContinuousProgress {
max_gap: Duration,
last_frame: Option<tokio::time::Instant>,
}
impl ContinuousProgress {
#[must_use]
pub fn new(max_gap: Duration) -> Self { Self { max_gap, last_frame: None } }
pub fn frame(&mut self, now: tokio::time::Instant) -> Result<(), TestkitError> {
if self.last_frame.is_some_and(|previous| now.duration_since(previous) > self.max_gap) {
return Err(TestkitError::Protocol("stream progress gap exceeded assertion bound".to_owned()));
}
self.last_frame = Some(now);
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(start_paused = true)]
async fn deadline_remains_anchored_when_later_events_arrive() {
let mut timeline = Timeline::default();
let key = EventKey { command_id: "start-b".to_owned(), event: "sent".to_owned() };
timeline.record(key.clone()).unwrap();
tokio::time::advance(Duration::from_secs(2)).await;
assert!(timeline.require_before(&key, Duration::from_secs(1), tokio::time::Instant::now()).is_err());
}
#[tokio::test(start_paused = true)]
async fn continuous_progress_rejects_long_gaps() {
let mut progress = ContinuousProgress::new(Duration::from_secs(1));
progress.frame(tokio::time::Instant::now()).unwrap();
tokio::time::advance(Duration::from_secs(2)).await;
assert!(progress.frame(tokio::time::Instant::now()).is_err());
}
}
tools/tuliprox-testkit/src/secret.rs
+20
View File
@@ -0,0 +1,20 @@
use crate::TestkitError;
pub fn environment(name: &str) -> Result<String, TestkitError> {
if name.trim().is_empty() {
return Err(TestkitError::Configuration("secret environment variable name is empty".to_owned()));
}
std::env::var(name)
.map_err(|_| TestkitError::Configuration(format!("required secret environment variable {name} is not set")))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn missing_variable_does_not_expose_a_value() {
let error = environment("TULIPROX_TESTKIT_MISSING_SECRET").unwrap_err().to_string();
assert!(error.contains("TULIPROX_TESTKIT_MISSING_SECRET"));
}
}
tools/tuliprox-testkit/src/shared.rs
+53
View File
@@ -0,0 +1,53 @@
use crate::TestkitError;
use std::collections::{HashMap, HashSet};
#[derive(Debug, Default)]
pub struct SharedStreamModel {
subscribers: HashMap<u32, HashSet<String>>,
}
impl SharedStreamModel {
/// Returns true only when this subscriber causes a new origin connection.
pub fn subscribe(&mut self, marker: u32, playback_id: String) -> Result<bool, TestkitError> {
let subscribers = self.subscribers.entry(marker).or_default();
if !subscribers.insert(playback_id) {
return Err(TestkitError::Protocol("shared-stream subscriber was registered twice".to_owned()));
}
Ok(subscribers.len() == 1)
}
/// Returns true only when the last subscriber closes the origin connection.
pub fn unsubscribe(&mut self, marker: u32, playback_id: &str) -> Result<bool, TestkitError> {
let subscribers = self
.subscribers
.get_mut(&marker)
.ok_or_else(|| TestkitError::Configuration(format!("shared stream marker {marker} is not active")))?;
if !subscribers.remove(playback_id) {
return Err(TestkitError::Configuration(format!(
"playback {playback_id} is not a shared-stream subscriber"
)));
}
let last = subscribers.is_empty();
if last {
self.subscribers.remove(&marker);
}
Ok(last)
}
#[must_use]
pub fn subscriber_count(&self, marker: u32) -> usize { self.subscribers.get(&marker).map_or(0, HashSet::len) }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_first_and_last_subscriber_change_origin_lifetime() {
let mut model = SharedStreamModel::default();
assert!(model.subscribe(17, "a".to_owned()).unwrap());
assert!(!model.subscribe(17, "b".to_owned()).unwrap());
assert!(!model.unsubscribe(17, "a").unwrap());
assert!(model.unsubscribe(17, "b").unwrap());
}
}
tools/tuliprox-testkit/src/transport.rs
+372
View File
@@ -0,0 +1,372 @@
use crate::{
protocol::{AgentId, AgentMessage, Command, Envelope, PlaybackEvent, RunId},
TestkitError,
};
use axum::{
extract::{
ws::{Message, WebSocket},
State, WebSocketUpgrade,
},
response::{IntoResponse, Response},
};
use futures::{SinkExt, StreamExt};
use std::{
collections::{HashMap, HashSet},
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
time::Duration,
};
use tokio::{
sync::{mpsc, Mutex},
time::Instant,
};
use tokio_tungstenite::{connect_async, tungstenite::protocol::Message as ClientMessage};
pub type ControllerState = Arc<SharedControllerState>;
pub struct SharedControllerState {
run_id: RunId,
generation: u64,
agents: Mutex<HashMap<AgentId, mpsc::Sender<Envelope<Command>>>>,
ready_agents: Mutex<HashSet<AgentId>>,
last_seen: Mutex<HashMap<AgentId, Instant>>,
events: Mutex<Vec<Envelope<AgentMessage>>>,
next_controller_sequence: AtomicU64,
}
impl SharedControllerState {
#[must_use]
pub fn new(run_id: RunId, generation: u64) -> Self {
Self {
run_id,
generation,
agents: Mutex::new(HashMap::new()),
ready_agents: Mutex::new(HashSet::new()),
last_seen: Mutex::new(HashMap::new()),
events: Mutex::new(Vec::new()),
next_controller_sequence: AtomicU64::new(1),
}
}
#[must_use]
pub fn next_command_sequence(&self) -> u64 { self.next_controller_sequence.fetch_add(1, Ordering::Relaxed) }
#[must_use]
pub fn run_id(&self) -> RunId { self.run_id.clone() }
pub async fn dispatch(&self, agent_id: &AgentId, command: Envelope<Command>) -> Result<(), TestkitError> {
command.validate()?;
if command.run_id != self.run_id || command.run_generation != self.generation {
return Err(TestkitError::Protocol("command does not belong to the active run generation".to_owned()));
}
let sender = self
.agents
.lock()
.await
.get(agent_id)
.cloned()
.ok_or_else(|| TestkitError::Protocol(format!("agent {} is not connected", agent_id.0)))?;
sender.send(command).await.map_err(|_| TestkitError::Protocol("agent control channel closed".to_owned()))
}
pub async fn connected_agents(&self) -> Vec<AgentId> { self.agents.lock().await.keys().cloned().collect() }
pub async fn ready_agents(&self) -> Vec<AgentId> {
const HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(15);
let now = Instant::now();
let ready = self.ready_agents.lock().await;
let seen = self.last_seen.lock().await;
ready
.iter()
.filter(|agent| {
seen.get(*agent).is_some_and(|last_seen| now.duration_since(*last_seen) <= HEARTBEAT_TIMEOUT)
})
.cloned()
.collect()
}
pub async fn take_events(&self) -> Vec<Envelope<AgentMessage>> { std::mem::take(&mut *self.events.lock().await) }
/// Removes only an event belonging to the requested playback. Events for
/// concurrent starts remain queued for their own waiter.
pub async fn take_playback_event(&self, playback_id: &str) -> Option<PlaybackEvent> {
let mut events = self.events.lock().await;
let position = events.iter().position(|envelope| {
matches!(
&envelope.payload,
AgentMessage::Event {
event: PlaybackEvent::FirstValidFrame { playback_id: observed, .. }
| PlaybackEvent::HeadersReceived { playback_id: observed, .. }
| PlaybackEvent::Progress { playback_id: observed, .. }
| PlaybackEvent::Terminal { playback_id: observed, .. },
} if observed.0 == playback_id
)
})?;
match events.remove(position).payload {
AgentMessage::Event { event } => Some(event),
AgentMessage::Hello { .. } | AgentMessage::Ready | AgentMessage::Heartbeat => None,
}
}
}
pub async fn controller_websocket(State(state): State<ControllerState>, websocket: WebSocketUpgrade) -> Response {
std::future::ready(()).await;
websocket.on_upgrade(move |socket| serve_agent(socket, state)).into_response()
}
async fn serve_agent(socket: WebSocket, state: ControllerState) {
let (mut sink, mut stream) = socket.split();
let Some(Ok(Message::Text(first))) = stream.next().await else { return };
let Ok(hello) = serde_json::from_str::<Envelope<AgentMessage>>(&first) else { return };
if hello.validate().is_err() || hello.run_id != state.run_id || hello.run_generation != state.generation {
return;
}
let AgentMessage::Hello { .. } = hello.payload else { return };
let agent_id = hello.agent_id;
let (tx, mut rx) = mpsc::channel(32);
state.agents.lock().await.insert(agent_id.clone(), tx);
state.last_seen.lock().await.insert(agent_id.clone(), Instant::now());
loop {
tokio::select! {
Some(command) = rx.recv() => {
let Ok(encoded) = serde_json::to_string(&command) else { break };
if sink.send(Message::Text(encoded.into())).await.is_err() { break; }
}
incoming = stream.next() => match incoming {
Some(Ok(Message::Close(_)) | Err(_)) | None => break,
Some(Ok(Message::Text(text))) => {
if let Ok(event) = serde_json::from_str::<Envelope<AgentMessage>>(&text) {
if event.validate().is_ok()
&& event.run_id == state.run_id
&& event.run_generation == state.generation
&& event.agent_id == agent_id
{
state.last_seen.lock().await.insert(agent_id.clone(), Instant::now());
match event.payload {
AgentMessage::Ready => {
state.ready_agents.lock().await.insert(agent_id.clone());
}
AgentMessage::Event { .. } => state.events.lock().await.push(event),
AgentMessage::Hello { .. } | AgentMessage::Heartbeat => {}
}
}
}
}
_ => {}
},
}
}
state.agents.lock().await.remove(&agent_id);
state.ready_agents.lock().await.remove(&agent_id);
state.last_seen.lock().await.remove(&agent_id);
}
pub struct AgentControlConnection {
commands: mpsc::Receiver<Envelope<Command>>,
events: mpsc::Sender<Envelope<AgentMessage>>,
}
impl AgentControlConnection {
pub async fn connect(controller_url: &str, hello: Envelope<AgentMessage>) -> Result<Self, TestkitError> {
hello.validate()?;
if !matches!(hello.payload, AgentMessage::Hello { .. }) {
return Err(TestkitError::Protocol("first agent message must be hello".to_owned()));
}
let (mut socket, _) =
connect_async(controller_url).await.map_err(|error| TestkitError::Protocol(error.to_string()))?;
let encoded = serde_json::to_string(&hello).map_err(|error| TestkitError::Protocol(error.to_string()))?;
socket
.send(ClientMessage::Text(encoded.into()))
.await
.map_err(|error| TestkitError::Protocol(error.to_string()))?;
let (mut sink, mut stream) = socket.split();
let (command_tx, command_rx) = mpsc::channel(32);
let (event_tx, mut event_rx) = mpsc::channel::<Envelope<AgentMessage>>(32);
tokio::spawn(async move {
while let Some(event) = event_rx.recv().await {
let Ok(encoded) = serde_json::to_string(&event) else { break };
if sink.send(ClientMessage::Text(encoded.into())).await.is_err() {
break;
}
}
});
tokio::spawn(async move {
while let Some(message) = stream.next().await {
let Ok(message) = message else { break };
match message {
ClientMessage::Text(text) => {
let Ok(command) = serde_json::from_str::<Envelope<Command>>(&text) else { continue };
if command.validate().is_err() || command_tx.send(command).await.is_err() {
break;
}
}
ClientMessage::Close(_) => break,
_ => {}
}
}
});
Ok(Self { commands: command_rx, events: event_tx })
}
pub async fn next_command(&mut self) -> Result<Option<Envelope<Command>>, TestkitError> {
Ok(self.commands.recv().await)
}
pub async fn send_event(&self, event: Envelope<AgentMessage>) -> Result<(), TestkitError> {
event.validate()?;
self.events.send(event).await.map_err(|_| TestkitError::Protocol("agent event channel closed".to_owned()))
}
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{routing::get, Router};
#[tokio::test]
async fn unknown_agent_cannot_receive_commands() {
let state = SharedControllerState::new(RunId::new("run"), 1);
let command = Envelope {
schema_version: 1,
run_id: RunId::new("run"),
run_generation: 1,
message_id: "command".to_owned(),
agent_id: AgentId::new("controller"),
agent_boot_id: "boot".to_owned(),
source_sequence: 1,
caused_by_command_id: None,
local_elapsed_nanos: 0,
payload: Command::StopAll,
};
assert!(state.dispatch(&AgentId::new("missing"), command).await.is_err());
}
#[tokio::test]
async fn agent_receives_generation_bound_command_over_websocket() {
let state: ControllerState = Arc::new(SharedControllerState::new(RunId::new("run"), 1));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let app = Router::new().route("/agent", get(controller_websocket)).with_state(state.clone());
let server = tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
let agent_id = AgentId::new("agent");
let hello = Envelope {
schema_version: 1,
run_id: RunId::new("run"),
run_generation: 1,
message_id: "hello".to_owned(),
agent_id: agent_id.clone(),
agent_boot_id: "boot".to_owned(),
source_sequence: 1,
caused_by_command_id: None,
local_elapsed_nanos: 0,
payload: AgentMessage::Hello {
hostname: "test".to_owned(),
supported_protocols: vec!["ts".to_owned()],
maximum_listeners: 1,
},
};
let mut agent = AgentControlConnection::connect(&format!("ws://{address}/agent"), hello).await.unwrap();
for _ in 0..10 {
if !state.connected_agents().await.is_empty() {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
agent
.send_event(Envelope {
schema_version: 1,
run_id: RunId::new("run"),
run_generation: 1,
message_id: "ready".to_owned(),
agent_id: agent_id.clone(),
agent_boot_id: "boot".to_owned(),
source_sequence: 2,
caused_by_command_id: None,
local_elapsed_nanos: 0,
payload: AgentMessage::Ready,
})
.await
.unwrap();
for _ in 0..10 {
if state.ready_agents().await.contains(&agent_id) {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert!(state.ready_agents().await.contains(&agent_id));
let command = Envelope {
schema_version: 1,
run_id: RunId::new("run"),
run_generation: 1,
message_id: "stop".to_owned(),
agent_id: AgentId::new("controller"),
agent_boot_id: "controller-boot".to_owned(),
source_sequence: 1,
caused_by_command_id: None,
local_elapsed_nanos: 0,
payload: Command::StopAll,
};
state.dispatch(&agent_id, command).await.unwrap();
assert!(matches!(agent.next_command().await, Ok(Some(Envelope { payload: Command::StopAll, .. }))));
agent
.send_event(Envelope {
schema_version: 1,
run_id: RunId::new("run"),
run_generation: 1,
message_id: "event".to_owned(),
agent_id,
agent_boot_id: "boot".to_owned(),
source_sequence: 3,
caused_by_command_id: None,
local_elapsed_nanos: 0,
payload: AgentMessage::Event {
event: crate::protocol::PlaybackEvent::Terminal {
playback_id: crate::protocol::PlaybackId::new("playback"),
outcome: "passed".to_owned(),
typed_outcome: None,
},
},
})
.await
.unwrap();
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
assert_eq!(state.take_events().await.len(), 1);
server.abort();
}
#[tokio::test]
async fn waiting_for_one_playback_keeps_other_playback_events() {
let state = SharedControllerState::new(RunId::new("run"), 1);
for playback_id in ["first", "second"] {
state.events.lock().await.push(Envelope {
schema_version: 1,
run_id: RunId::new("run"),
run_generation: 1,
message_id: format!("event-{playback_id}"),
agent_id: AgentId::new("agent"),
agent_boot_id: "boot".to_owned(),
source_sequence: 1,
caused_by_command_id: None,
local_elapsed_nanos: 0,
payload: AgentMessage::Event {
event: PlaybackEvent::FirstValidFrame {
playback_id: crate::protocol::PlaybackId::new(playback_id),
sequence: 0,
},
},
});
}
assert!(matches!(
state.take_playback_event("second").await,
Some(PlaybackEvent::FirstValidFrame { playback_id, .. }) if playback_id.0 == "second"
));
assert!(matches!(
state.take_playback_event("first").await,
Some(PlaybackEvent::FirstValidFrame { playback_id, .. }) if playback_id.0 == "first"
));
}
}
tools/tuliprox-testkit/src/vod.rs
+345
View File
@@ -0,0 +1,345 @@
use crate::TestkitError;
use bytes::Bytes;
use futures::Stream;
use reqwest::{header, StatusCode};
use std::{pin::Pin, time::Duration};
use tokio::sync::oneshot;
pub const DEFAULT_VOD_SIZE: u64 = 128 * 1024 * 1024; // 128 MiB virtual media file
const BLOCK_SIZE: usize = 1024;
pub async fn read_range(url: &str, start: u64, end_inclusive: u64) -> Result<Vec<u8>, TestkitError> {
if start > end_inclusive {
return Err(TestkitError::Configuration("VOD range start exceeds end".to_owned()));
}
let range = format!("bytes={start}-{end_inclusive}");
let response = reqwest::Client::new().get(url).header(header::RANGE, range).send().await?;
if response.status() != StatusCode::PARTIAL_CONTENT {
return Err(TestkitError::Protocol(format!("VOD range returned {}, expected 206", response.status())));
}
let expected_range = format!("bytes {start}-{end_inclusive}/");
let content_range = response
.headers()
.get(header::CONTENT_RANGE)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| TestkitError::Protocol("VOD range response has no Content-Range".to_owned()))?;
if !content_range.starts_with(&expected_range) {
return Err(TestkitError::Protocol(format!("unexpected VOD Content-Range {content_range}")));
}
let body = response.bytes().await?.to_vec();
let expected_len = usize::try_from(end_inclusive.saturating_sub(start).saturating_add(1))
.map_err(|_| TestkitError::Configuration("VOD range is too large for this host".to_owned()))?;
if body.len() != expected_len {
return Err(TestkitError::Protocol(format!("VOD range has {} bytes, expected {expected_len}", body.len())));
}
Ok(body)
}
#[must_use]
pub fn parse_range_header(value: &str, total_length: u64) -> Option<(u64, u64)> {
let bytes = value.strip_prefix("bytes=")?;
if let Some(suffix) = bytes.strip_prefix('-') {
let suffix_len = suffix.parse::<u64>().ok()?;
if suffix_len == 0 || suffix_len > total_length {
return Some((0, total_length));
}
let start = total_length.saturating_sub(suffix_len);
return Some((start, total_length));
}
let (start_str, end_str) = bytes.split_once('-')?;
let start = start_str.parse::<u64>().ok()?;
if start >= total_length {
return None;
}
let end = if end_str.is_empty() {
total_length
} else {
let parsed_end = end_str.parse::<u64>().ok()?;
parsed_end.min(total_length.saturating_sub(1)).saturating_add(1)
};
if start >= end {
return None;
}
Some((start, end))
}
#[must_use]
pub fn parse_content_range(value: &str) -> Option<(u64, u64, Option<u64>)> {
let bytes = value.strip_prefix("bytes ")?;
let (range_part, total_part) = bytes.split_once('/')?;
let (start_str, end_str) = range_part.split_once('-')?;
let start = start_str.parse::<u64>().ok()?;
let end = end_str.parse::<u64>().ok()?;
let total = if total_part == "*" { None } else { Some(total_part.parse::<u64>().ok()?) };
Some((start, end, total))
}
#[must_use]
pub fn generate_deterministic_block(object: &str, block_index: u64) -> [u8; BLOCK_SIZE] {
let seed = blake3::hash(object.as_bytes());
let block_hash = blake3::keyed_hash(seed.as_bytes(), &block_index.to_le_bytes());
let mut block = [0_u8; BLOCK_SIZE];
block[0..8].copy_from_slice(&block_index.to_be_bytes());
let hash_bytes = block_hash.as_bytes();
for chunk in block[8..].chunks_mut(32) {
let len = chunk.len().min(32);
chunk.copy_from_slice(&hash_bytes[..len]);
}
block
}
#[must_use]
pub fn generate_deterministic_slice(object: &str, offset: u64, len: usize) -> Vec<u8> {
let mut out = Vec::with_capacity(len);
let mut current_offset = offset;
let end_offset = offset.saturating_add(len as u64);
while current_offset < end_offset {
let block_index = current_offset / (BLOCK_SIZE as u64);
let in_block = usize::try_from(current_offset % (BLOCK_SIZE as u64)).unwrap_or(0);
let block = generate_deterministic_block(object, block_index);
let remaining_in_block = BLOCK_SIZE.saturating_sub(in_block);
let remaining_requested = usize::try_from(end_offset.saturating_sub(current_offset)).unwrap_or(0);
let take = remaining_in_block.min(remaining_requested);
out.extend_from_slice(&block[in_block..in_block.saturating_add(take)]);
current_offset = current_offset.saturating_add(take as u64);
}
out
}
#[must_use]
pub fn verify_deterministic_slice(object: &str, offset: u64, data: &[u8]) -> bool {
let expected = generate_deterministic_slice(object, offset, data.len());
data == expected.as_slice()
}
#[derive(Debug, Clone)]
pub struct VodStreamOptions {
pub object: String,
pub start_offset: u64,
pub length: u64,
pub bitrate: u32,
pub chunk_size: usize,
pub stall_ms: Option<u64>,
pub abort_after_bytes: Option<u64>,
}
impl Default for VodStreamOptions {
fn default() -> Self {
Self {
object: "movie.mkv".to_owned(),
start_offset: 0,
length: DEFAULT_VOD_SIZE,
bitrate: 0,
chunk_size: 64 * 1024,
stall_ms: None,
abort_after_bytes: None,
}
}
}
struct VodStreamState {
options: VodStreamOptions,
evict_rx: oneshot::Receiver<()>,
evicted_flag: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
current_offset: u64,
end_offset: u64,
total_emitted: u64,
interval: Option<tokio::time::Interval>,
initial_stall_done: bool,
}
pub fn create_vod_stream(
options: VodStreamOptions,
evict_rx: oneshot::Receiver<()>,
) -> Pin<Box<dyn Stream<Item = Result<Bytes, std::convert::Infallible>> + Send>> {
create_vod_stream_with_evicted(options, evict_rx, None)
}
pub fn create_vod_stream_with_evicted(
options: VodStreamOptions,
evict_rx: oneshot::Receiver<()>,
evicted_flag: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
) -> Pin<Box<dyn Stream<Item = Result<Bytes, std::convert::Infallible>> + Send>> {
let current_offset = options.start_offset;
let end_offset = options.start_offset.saturating_add(options.length);
let interval = if options.bitrate > 0 {
let bits_per_chunk = (options.chunk_size as u128).saturating_mul(8);
let nanos =
bits_per_chunk.saturating_mul(1_000_000_000).checked_div(u128::from(options.bitrate.max(1))).unwrap_or(1);
let duration = Duration::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX).max(1));
let mut interval = tokio::time::interval(duration);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
Some(interval)
} else {
None
};
let state = VodStreamState {
options,
evict_rx,
evicted_flag,
current_offset,
end_offset,
total_emitted: 0,
interval,
initial_stall_done: false,
};
Box::pin(futures::stream::unfold(state, |mut state| async move {
if !state.initial_stall_done {
state.initial_stall_done = true;
if let Some(stall) = state.options.stall_ms {
tokio::time::sleep(Duration::from_millis(stall)).await;
}
}
if state.current_offset >= state.end_offset {
return None;
}
if let Some(abort_limit) = state.options.abort_after_bytes {
if state.total_emitted >= abort_limit {
return None;
}
}
let remaining = usize::try_from(state.end_offset.saturating_sub(state.current_offset)).unwrap_or(0);
let mut take = state.options.chunk_size.min(remaining);
if let Some(abort_limit) = state.options.abort_after_bytes {
let bytes_until_abort = usize::try_from(abort_limit.saturating_sub(state.total_emitted)).unwrap_or(0);
take = take.min(bytes_until_abort);
}
if take == 0 {
return None;
}
if let Some(interval) = state.interval.as_mut() {
tokio::select! {
biased;
res = &mut state.evict_rx => {
if res.is_ok() {
if let Some(flag) = state.evicted_flag.as_ref() {
flag.store(true, std::sync::atomic::Ordering::Release);
}
}
return None;
}
_ = interval.tick() => {}
}
} else {
match state.evict_rx.try_recv() {
Ok(()) => {
if let Some(flag) = state.evicted_flag.as_ref() {
flag.store(true, std::sync::atomic::Ordering::Release);
}
return None;
}
Err(oneshot::error::TryRecvError::Closed) => return None,
Err(oneshot::error::TryRecvError::Empty) => {}
}
}
let chunk = generate_deterministic_slice(&state.options.object, state.current_offset, take);
state.current_offset = state.current_offset.saturating_add(take as u64);
state.total_emitted = state.total_emitted.saturating_add(take as u64);
Some((Ok(Bytes::from(chunk)), state))
}))
}
#[cfg(test)]
mod tests {
use super::*;
use tokio_stream::StreamExt;
#[test]
fn parses_open_and_closed_and_suffix_ranges() {
assert_eq!(parse_range_header("bytes=0-1023", 10_000), Some((0, 1024)));
assert_eq!(parse_range_header("bytes=100-", 10_000), Some((100, 10_000)));
assert_eq!(parse_range_header("bytes=-500", 10_000), Some((9500, 10_000)));
assert_eq!(parse_range_header("bytes=10000-", 10_000), None);
assert_eq!(parse_range_header("bytes=500-200", 10_000), None);
}
#[test]
fn deterministic_slice_verification_is_exact() {
let object = "test_vod.mkv";
let offset = 4096;
let slice = generate_deterministic_slice(object, offset, 2048);
assert_eq!(slice.len(), 2048);
assert!(verify_deterministic_slice(object, offset, &slice));
// Tampered byte must fail verification
let mut tampered = slice.clone();
tampered[10] ^= 0xFF;
assert!(!verify_deterministic_slice(object, offset, &tampered));
// Wrong offset must fail verification
assert!(!verify_deterministic_slice(object, offset + 1, &slice));
}
#[tokio::test]
async fn vod_stream_respects_abort_after_bytes() {
let options = VodStreamOptions {
object: "movie.mkv".to_owned(),
start_offset: 0,
length: 100_000,
bitrate: 0,
chunk_size: 1024,
stall_ms: None,
abort_after_bytes: Some(2500),
};
let (_tx, rx) = oneshot::channel();
let mut stream = create_vod_stream(options, rx);
let mut collected = Vec::new();
while let Some(Ok(chunk)) = stream.next().await {
collected.extend_from_slice(&chunk);
}
assert_eq!(collected.len(), 2500);
assert!(verify_deterministic_slice("movie.mkv", 0, &collected));
}
#[tokio::test]
async fn vod_stream_stops_on_eviction() {
let options = VodStreamOptions {
object: "movie.mkv".to_owned(),
start_offset: 0,
length: 1_000_000,
bitrate: 0,
chunk_size: 1024,
stall_ms: None,
abort_after_bytes: None,
};
let (tx, rx) = oneshot::channel();
let mut stream = create_vod_stream(options, rx);
// Read 1 chunk
let first = stream.next().await.unwrap().unwrap();
assert_eq!(first.len(), 1024);
// Send eviction signal
let _ = tx.send(());
// Next read must terminate
assert!(stream.next().await.is_none());
}
#[test]
fn parse_content_range_valid_and_invalid() {
assert_eq!(parse_content_range("bytes 0-65535/134217728"), Some((0, 65535, Some(134_217_728))));
assert_eq!(parse_content_range("bytes 100-200/*"), Some((100, 200, None)));
assert_eq!(parse_content_range("bytes 0-0/1"), Some((0, 0, Some(1))));
assert_eq!(parse_content_range("0-65535/134217728"), None);
assert_eq!(parse_content_range("bytes 0-65535"), None);
assert_eq!(parse_content_range("bytes invalid-range/100"), None);
}
}