diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1dd6cc2f3..6df54b06d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,7 +4,7 @@ on: # push: # branches: [] pull_request: - branches: [develop] + branches: [develop, experimental] # Avoid running the same CI multiple times for the same PR concurrency: @@ -17,10 +17,10 @@ jobs: name: Markdown Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Lint Markdown files - uses: DavidAnson/markdownlint-cli2-action@v24 + uses: DavidAnson/markdownlint-cli2-action@28a7e8bdb81fd8ad675883de92c758ab78e0ce10 # v24 with: globs: '**/*.md' @@ -29,7 +29,7 @@ jobs: name: Format & Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Install Nightly Rust uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 @@ -57,10 +57,10 @@ jobs: name: Documentation Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Install mdBook - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@9114bf4d891761788c546334fd37538eae1bf8b3 # v2 with: tool: mdbook @@ -71,20 +71,20 @@ jobs: name: Frontend Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Install Rust Tools - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@9114bf4d891761788c546334fd37538eae1bf8b3 # v2 with: tool: trunk,wasm-bindgen-cli,mdbook - name: Install Stable Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable with: targets: wasm32-unknown-unknown - name: Rust Cache - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2 with: cache-targets: "true" @@ -105,15 +105,15 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Install Stable Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable - name: Rust Cache - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2 # Non-test code only. `--all-targets` would additionally compile the # test modules, ~10 of which still use ungated `std::os::unix` @@ -129,14 +129,54 @@ jobs: name: Build & Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Install Stable Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable - name: Rust Cache # This magic step saves compiled dependencies between runs - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2 - name: Run Tests run: make test + + admission: + name: Admission & Provider Lifecycle + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + + - name: Install Stable Rust + uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable + + - name: Rust Cache + uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2 + + - name: Verify admission and provider lifecycle invariants + run: make admission-test + + testkit: + name: Testkit Unit & E2E Scenarios + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + + - name: Install Stable Rust + uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable + + - name: Rust Cache + uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2 + + - name: Run Testkit Unit Tests + run: make testkit-test + + - name: Run Testkit E2E Scenarios + run: make testkit-e2e + + - name: Upload Testkit Reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: testkit-report + path: testkit-report/ diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index edfe1f7e6..2bfa82aeb 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -10,7 +10,7 @@ on: branch: type: choice description: Target branch - options: [develop, master] + options: [develop, experimental, master] default: develop choice: type: choice @@ -18,7 +18,7 @@ on: options: [none, major, minor, patch] default: none push: - branches: [develop, master] + branches: [develop, experimental, master] # Avoid multiple parallel builds for the same branch to prevent race conditions concurrency: @@ -29,7 +29,7 @@ jobs: docker-build: runs-on: ubuntu-latest if: >- - (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && + (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/experimental' || github.ref == 'refs/heads/master') && (github.event_name != 'push' || github.actor != 'github-actions[bot]') steps: @@ -67,12 +67,13 @@ jobs: if [[ "${{ github.event_name }}" == "push" ]]; then case "${{ github.ref_name }}" in develop) ./bin/inc_version.sh ;; + experimental) echo "πŸ”¬ Experimental builds do not bump versions; keeping version from develop." ;; master) echo "Master builds use the version already committed in the merged PR; skipping version bump." ;; *) echo "Unsupported push branch: ${{ github.ref_name }}" >&2; exit 1 ;; esac else - if [[ "${{ github.event.inputs.branch }}" == "master" ]]; then - echo "Manual master builds use the version already committed on master; skipping version bump." + if [[ "${{ github.event.inputs.branch }}" == "master" || "${{ github.event.inputs.branch }}" == "experimental" ]]; then + echo "Manual ${{ github.event.inputs.branch }} builds skip version bump." else case "${{ github.event.inputs.choice }}" in major) ./bin/inc_version.sh m ;; @@ -147,7 +148,7 @@ jobs: - name: Commit version bump if: >- (github.event_name == 'push' && github.ref == 'refs/heads/develop') || - (github.event_name == 'workflow_dispatch' && github.event.inputs.branch != 'master') + (github.event_name == 'workflow_dispatch' && github.event.inputs.branch == 'develop' && github.event.inputs.choice != 'none') id: version_commit run: | git config user.name "euzu" @@ -220,7 +221,7 @@ jobs: success() && ( (github.event_name == 'push' && github.ref == 'refs/heads/develop') || - (github.event_name == 'workflow_dispatch' && github.event.inputs.branch != 'master') + (github.event_name == 'workflow_dispatch' && github.event.inputs.branch == 'develop') ) && steps.version_commit.outputs.did_commit == 'true' run: | diff --git a/.gitignore b/.gitignore index d6e7c1144..06f78230a 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,6 @@ backend/app/Cargo.lock .env.* !.env.example +# Testkit report artifacts +/testkit-report/ + diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b2b620cc..80147a3bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,12 @@ ## 🌟 New Features +- **Runtime liveness watchdog.** An optional heartbeat/watchdog detects a wedged + async runtime (process alive, scheduler no longer making progress, logs stop) and logs a diagnostic snapshot with + runtime metrics and a per-thread `/proc/self/task` inventory. It is opt-in and off by default (`TULIPROX_WATCHDOG=1` + to observe, `=2` to also restart the process on a confirmed stall), and exposes its state through the `/healthcheck` + `runtime` object. + - **`.env` file support for secrets and environment variables:** Tuliprox now automatically loads environment variables from a `.env` file at startup. - **Discovery order:** searches `--env-file ` (or `-e`), `TULIPROX_ENV_FILE`, then `/.env` @@ -985,6 +991,41 @@ ## πŸ› Fixes +- **Provider priority was ignored and a second concurrent client failed with a source error while capacity was free.** + Provider-slot reservations were granted as soon as a playback opened a provider, so an HLS/DASH entry that only ever + served a manifest β€” or a player that retried its manifest and gave up β€” still held a reservation for the whole + `hls_session_ttl_secs` window. Because HLS entry session tokens carry a per-attempt suffix, every retry created a + *separate* reservation under a *separate* owner. Those reservations were foreign to one another and to unrelated + clients, so a higher-priority provider was skipped as reserved even after the active connection counters had reached + zero and playback fell through to a lower-priority alias. Clients sharing one reverse-proxy socket hit this + constantly, which looked like exhausted capacity or socket-based reservation collisions. + + Reservations are now provider slot leases with an explicit confirmation step. A lease starts unconfirmed, pins the + provider for its own playback, and blocks nobody else. Only real media delivery confirms it β€” the first provider media + byte forwarded to a client as an `OK`/`206` media response. Manifest, HEAD, key, map and error fetches never confirm a + lease β€” and only a confirmed lease with a configured reconnect + window reserves capacity against other playbacks. Unconfirmed leases expire after a short startup deadline regardless + of TTL, so abandoned starts and manifest-retry loops no longer accumulate. One owner holds at most one lease, so + repeated requests of the same playback reuse their slot instead of stacking a reservation per attempt. + + Lease end is outcome-driven: a clean finish of a reconnect-capable playback (HLS, DASH, VOD, series, catchup) keeps the + slot as an idle lease for its window, while provider failure, preemption, kick and timeout release capacity + immediately. The `Skipping reserved provider ... for ` message is replaced by a structured decision log + carrying the reason, current and maximum connections, foreign reserved slots, and the active/starting/idle slot split, + so a fallback decision can be audited without inferring it from a socket address. + + Playback cleanup is now request-specific. Parallel segment, range and reconnect requests carry independent request + identities, while a provider-binding generation prevents delayed cleanup from releasing a newer successor binding. + Provider allocations are owned by drop guards until they are explicitly transferred, closing cancellation gaps during + provider open, grace handling and HLS origin refresh. + + Shared MPEG-TS subscribers now have identities independent of their transport socket, so clients sharing one reverse + proxy connection cannot replace or cancel one another. Subscriber queues enforce byte and chunk budgets, slow clients + have progress deadlines, and incomplete burst replay ends only the affected subscriber instead of skipping into live + delivery. Cleanup admission is bounded by the new `reverse_proxy.stream.cleanup_queue_capacity` setting (default + `4096`); mandatory HLS cleanup uses a reserved control lane. Graceful shutdown closes admission, releases active + claims and provider leases, and waits for owned streaming and cleanup workers to finish. + - **PTT title parsing: fixed panics on multi-byte UTF-8 character boundaries (e.g. en-dash `–`).** During title metadata parsing (such as background VOD/series metadata enrichment), previous match indices recorded from earlier handlers (e.g. `year`) could become stale after preceding handlers removed matched substrings in-place with @@ -1295,6 +1336,14 @@ ## βš™οΈ New Settings +- **Runtime diagnostics (environment variables)**: + - `TULIPROX_WATCHDOG` (default unset = off) is a mode selector: `1` (`true`/`on`/`yes`/`enabled`) observes and logs + stalls, `2` (`restart`) additionally exits the process after the stall persists so a supervisor restarts it. + - `TULIPROX_WATCHDOG_HEARTBEAT_MS` (default `1000`), `TULIPROX_WATCHDOG_STALL_MS` (default `10000`), + `TULIPROX_WATCHDOG_RELOG_MS` (default `30000`) and `TULIPROX_WATCHDOG_RESTART_GRACE_MS` (default `30000`): heartbeat + cadence, stall threshold, re-log interval and restart grace. + - `TULIPROX_TOKIO_CONSOLE` (default unset): set to `1` to start the console subscriber in a `tokio-console` build. + - **source.yml (target `options`)**: - Added optional `clear_invalid_epg_ids` (`bool`, default `false`) to clear unresolved live-channel EPG IDs after EPG matching and final mappings without removing playlist entries. The legacy name `required_epg` is accepted while diff --git a/Cargo.lock b/Cargo.lock index 39e3a91ee..c47091847 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" @@ -422,6 +422,29 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bon" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease 0.3.0", + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "brotli" version = "8.0.4" @@ -818,8 +841,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -836,17 +869,41 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.4", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", "quote", "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.4", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -917,9 +974,6 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", -] [[package]] name = "derive_builder" @@ -936,7 +990,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.119", @@ -2531,9 +2585,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -2944,6 +2998,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.4", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3472,9 +3536,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -4007,12 +4071,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "libc", "num-conv", "num_threads", @@ -4024,15 +4087,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -4840,6 +4903,33 @@ dependencies = [ "url", ] +[[package]] +name = "tuliprox-testkit" +version = "3.3.113" +dependencies = [ + "axum", + "base64 0.23.1", + "blake3", + "bytes", + "clap", + "crc32fast", + "futures", + "hyper", + "hyper-util", + "pin-project-lite", + "reqwest", + "serde", + "serde-saphyr", + "serde_json", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tokio-tungstenite", + "tower-service", + "url", +] + [[package]] name = "tungstenite" version = "0.29.0" @@ -4947,12 +5037,12 @@ dependencies = [ [[package]] name = "vergen" -version = "9.1.0" +version = "10.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b849a1f6d8639e8de261e81ee0fc881e3e3620db1af9f2e0da015d4382ceaf75" +checksum = "d7081a999aada4e01c8ee04d32254e021e317dd051a250169305fbf2acf21824" dependencies = [ "anyhow", - "derive_builder", + "bon", "rustversion", "time", "vergen-lib", @@ -4960,12 +5050,12 @@ dependencies = [ [[package]] name = "vergen-lib" -version = "9.1.0" +version = "10.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34a29ba7e9c59e62f229ae1932fb1b8fb8a6fdcc99215a641913f5f5a59a569" +checksum = "1343066736be577e40cf72914ecd68b369f7033ce0f7db6b25a6d6f7fbb3ed70" dependencies = [ "anyhow", - "derive_builder", + "bon", "rustversion", ] @@ -5639,7 +5729,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08b883d84a035f57519d057f65a2a07ae25f1884ad485e1a9a523c9d880c1ad" dependencies = [ - "prettyplease", + "prettyplease 0.2.37", "proc-macro-error", "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index ba5fbe21a..6e72e053f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["backend/*", "frontend", "shared"] +members = ["backend/*", "frontend", "shared", "tools/tuliprox-testkit"] resolver = "2" [workspace.package] @@ -29,7 +29,6 @@ shared = { version = "3", path = "shared" } tuliprox-hls = { version = "3", path = "backend/hls" } tuliprox-auth = { version = "3", path = "backend/auth" } tuliprox-dvr = { version = "3", path = "backend/dvr" } -tuliprox = { version = "3", path = "backend/app" } tuliprox-mpegts = { version = "3", path = "backend/mpegts" } tuliprox-iptv = { version = "3", path = "backend/iptv" } tuliprox-parser = { version = "3", path = "backend/parser" } diff --git a/Makefile b/Makefile index d68fc8fc5..b173a43aa 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ OS := $(shell uname -s) ARCH := $(shell uname -m) # Paths +PROJECT_DIR ?= $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) CARGO_HOME ?= $(HOME)/.cargo CARGO_BIN_DIR ?= $(CARGO_HOME)/bin @@ -34,6 +35,21 @@ BOLD := \033[1m CPU_COUNT := $(shell nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1) CARGO_BUILD_JOBS := $(CPU_COUNT) +# Support positional argument for serve: make serve +ifeq ($(firstword $(MAKECMDGOALS)),serve) + SETTINGS_ARG := $(word 2,$(MAKECMDGOALS)) + ifneq ($(SETTINGS_ARG),) + SETTINGS_FOLDER ?= $(SETTINGS_ARG) + EXTRA_GOALS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) + $(eval .PHONY: $(EXTRA_GOALS)) + $(eval $(EXTRA_GOALS):;@:) + endif +endif +SETTINGS_FOLDER ?= $(settings_folder) +ifeq ($(SETTINGS_FOLDER),) + SETTINGS_FOLDER := $(TULIPROX_HOME) +endif + .PHONY: help help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make $(AQUA)$(RESET)\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " $(AQUA)%-26s$(RESET) %s\n", $$1, $$2 } /^##@/ { printf "\n$(BOLD)%s$(RESET)\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -149,11 +165,66 @@ test: ## Run all workspace tests (Stable) β€” use detected CPU count for paralle @echo "==> Running tests (stable) with $(CPU_COUNT) jobs/threads" @TMPDIR="$${TMPDIR:-/tmp}" RUST_TEST_THREADS=$(CPU_COUNT) ./bin/test.sh -j$(CPU_COUNT) --workspace -- --test-threads=$(CPU_COUNT) +.PHONY: admission-test +admission-test: ## Verify admission, provider-slot, seek/reopen, shared-stream, and cleanup invariants + @echo "==> Running HTTP admission and provider lifecycle tests" + @$(CARGO_STABLE) test --package tuliprox api_utils::tests:: -- --test-threads=1 + @echo "==> Running client stream lifecycle tests" + @$(CARGO_STABLE) test --package tuliprox api::model::streams::active_client_stream::tests:: -- --test-threads=1 + @echo "==> Running session, eviction, and provider-slot lifecycle tests" + @$(CARGO_STABLE) test --package tuliprox-session -- --test-threads=1 + +.PHONY: testkit-test +testkit-test: ## Run unit tests in tuliprox-testkit + @echo "==> Running testkit unit tests" + @$(CARGO_STABLE) test --package tuliprox-testkit + +.PHONY: testkit-e2e +testkit-e2e: ## Run testkit E2E scenario suite against freshly built SUT + @echo "==> Building tuliprox and tuliprox-testkit" + @$(CARGO_STABLE) build --package tuliprox --package tuliprox-testkit + @echo "==> Executing testkit scenario suite" + @./bin/run-testkit-scenarios.sh + +.PHONY: testkit-scenario +testkit-scenario: ## Run one testkit scenario: make testkit-scenario SCENARIO= [RUN_ID=] + @if [ -z "$(SCENARIO)" ]; then \ + echo "❌ Error: SCENARIO is required."; \ + echo "Usage: make testkit-scenario SCENARIO="; \ + echo "Available scenarios:"; \ + ls test/fixtures/testkit/scenarios/*.yml | xargs -n1 basename | sed 's/\.yml$$//' | sed 's/^/ - /'; \ + exit 1; \ + fi + @if [ ! -f "test/fixtures/testkit/scenarios/$(SCENARIO).yml" ]; then \ + echo "❌ Unknown scenario: $(SCENARIO)"; \ + exit 1; \ + fi + @echo "==> Building tuliprox and tuliprox-testkit" + @$(CARGO_STABLE) build --package tuliprox --package tuliprox-testkit + @mkdir -p testkit-report/$(SCENARIO) + @echo "==> Running scenario $(SCENARIO)" + @TULIPROX_TESTKIT_SUT_BINARY="$(PROJECT_DIR)/target/debug/tuliprox" \ + ./target/debug/tuliprox-testkit controller \ + --scenario test/fixtures/testkit/scenarios/$(SCENARIO).yml \ + --report-directory testkit-report/$(SCENARIO) \ + $(if $(RUN_ID),--run-id $(RUN_ID),) + @echo "==> Report: testkit-report/$(SCENARIO)/summary.txt" + .PHONY: build build: ## Build the entire workspace in parallel using detected CPU count @echo "==> Building workspace with $(CARGO_BUILD_JOBS) jobs" @TMPDIR="$${TMPDIR:-/tmp}" $(CARGO_STABLE) build -j$(CARGO_BUILD_JOBS) --workspace +.PHONY: serve +serve: ## Run tuliprox server with settings folder: make serve + @if [ -z "$(SETTINGS_FOLDER)" ]; then \ + echo "❌ Error: Settings folder is required."; \ + echo "Usage: make serve OR make serve SETTINGS_FOLDER="; \ + exit 1; \ + fi + @echo "==> Starting tuliprox server with TULIPROX_HOME=$(SETTINGS_FOLDER)" + TULIPROX_HOME="$(SETTINGS_FOLDER)" $(CARGO) run --release --manifest-path $(PROJECT_DIR)/Cargo.toml --package tuliprox --bin tuliprox -- -s + .PHONY: architecture-check architecture-check: ## Verify workspace dependency direction @echo "==> Checking the architecture gate itself" @@ -180,6 +251,8 @@ fmt: ## Format all code using nightly rules (Compact) fmt-check: ## Check if code follows formatting rules (Nightly) @echo "==> Checking formatting (nightly)" $(CARGO_NIGHTLY) fmt --all -- --check + @echo "==> Checking git diff for whitespace errors" + git diff --check .PHONY: markdown-lint mdlint markdown-lint: ## Lint markdown files diff --git a/README.md b/README.md index 13bc97481..4a19e37f5 100644 --- a/README.md +++ b/README.md @@ -1,425 +1,505 @@ -## **tuliprox** - A Powerful IPTV Proxy & Playlist Processor +# **tuliprox** β€” Self-Hosted Media Gateway & Playlist Processor -`tuliprox` is a high-performance IPTV proxy and playlist processor written in Rust πŸ¦€. -It unifies M3U/M3U8 playlists, Xtream providers, Stalker/Ministra portals, Plex, Emby, Jellyfin and local media, -reshapes them into clean outputs, and serves them to IPTV players and media clients from one place. +`tuliprox` is a high-performance, self-hosted media gateway for bringing IPTV providers, media servers, playlists, EPG data, +and local media libraries together behind one clean and controllable interface. + +Instead of configuring every player against every upstream source, Tuliprox sits in the middle: it imports your authorized +media sources, normalizes and enriches their metadata, filters and reorganizes the catalog, manages provider connections, +and publishes the result in the formats your clients already understand. + +**One service. Multiple sources. Multiple users. Multiple output formats. Full control.** + +## ✨ Why Tuliprox? + +Tuliprox is built for more than playlist conversion. It is designed to become the central media gateway for a self-hosted setup: + +- **Unify fragmented sources** β€” combine Xtream, M3U/M3U8, Stalker/Ministra, Plex, Emby, Jellyfin, local media, staged catalogs, + and EPG data in one place. +- **Publish once, consume everywhere** β€” expose the same processed catalog as M3U, Xtream Codes API, HDHomeRun, or STRM without + maintaining separate setups. +- **Protect provider connections** β€” share streams, enforce connection limits, prioritize users, recover from upstream problems, + and avoid wasting provider slots on unnecessary background work. +- **Shape the catalog your way** β€” filter, rename, map, sort, merge, deduplicate, curate, and enrich content before it reaches users. +- **Operate multiple users safely** β€” assign targets, plans, output clusters, network restrictions, priorities, limits, and + server profiles per user. +- **Run it like infrastructure** β€” health and readiness probes, durable notifications, stream history, QoS snapshots, quality + guards, hot reloads, audit events, and optional runtime watchdog support are built in. +- **Keep the stack small** β€” Rust, a single application, embedded storage, and no mandatory external database. +- **Stay in control of your data** β€” self-hosted configuration, metadata, user management, recordings, and operational state. + +## πŸš€ At a Glance + +| Area | What Tuliprox gives you | +|----------------|--------------------------------------------------------------------------------------------------------------| +| **Inputs** | Xtream, M3U/M3U8, Stalker/Ministra, Plex, Emby, Jellyfin, local libraries, staged sources | +| **Outputs** | M3U/M3U8, Xtream Codes API, HDHomeRun, STRM | +| **Processing** | Filtering, mapping DSL, sorting, templates, EPG matching, metadata enrichment, curation | +| **Streaming** | Reverse proxy, redirects, shared MPEG-TS/HLS, HLS cache/prefetch, catchup proxying, failover | +| **Users** | Multi-user gateway, plans, connection limits, priorities, output clusters, category access, network policies | +| **Operations** | Web UI, REST API, scheduler, health/readiness probes, notifications, QoS, stream history, watchdog | +| **Storage** | Embedded B+Tree engine, WAL protection, mmap scans, compression, compaction β€” no external DB required | +| **Deployment** | Docker, Docker Compose templates, Raspberry Pi, NAS, VPS, x86 and ARM | + +> **Legal Notice** +> +> `tuliprox` does not provide, host, sell, or distribute media content or access credentials. +> Users are solely responsible for ensuring that they have the necessary rights and authorization to access, process, +> proxy, record, or redistribute any media sources configured with `tuliprox`. +> +> The software is intended for use with legally obtained and properly authorized content and services. ![tuliprox logo](https://github.com/user-attachments/assets/8ef9ea79-62ff-4298-978f-22326c5c3d02) -## Want to join the community - -[Join us on Discord](https://discord.gg/gkzCmWw9Tf) - -## License - -See [`LICENSE`](https://github.com/euzu/tuliprox/blob/develop/LICENSE). - ## πŸ† Key Features -### 1. Written in Rust β€” Maximum Performance, Minimal Footprint +### 1. Fast, Lightweight, and Built for 24/7 Operation -- Single binary β€” no Python, no Node.js, no Java, no runtime overhead -- **Extremely low CPU usage** β€” even with hundreds of concurrent streams -- **Very low RAM consumption** β€” runs comfortably on 256 MB, disk-based processing mode for even less -- Native async I/O with Tokio β€” thousands of concurrent streams without breaking a sweat -- **Rock-solid stability** β€” designed to run 24/7 for months without memory leaks, restarts, or degradation -- Runs on Raspberry Pi, tiny VPS, NAS, or any x86/ARM system -- No external database required β€” everything embedded +Tuliprox is written in Rust and designed to keep the runtime footprint small even when the catalog and number of connected clients grow. -### 2. Custom B+Tree Storage Engine β€” No External Database Needed +- Single native application β€” no Python, Node.js, Java, or mandatory database server +- Low CPU and memory overhead, including disk-based playlist processing for memory-constrained systems +- Runs comfortably on small servers and can operate with as little as 256 MB RAM depending on workload +- Tokio-based asynchronous I/O for highly concurrent streaming and background work +- Runs on Raspberry Pi, NAS, small VPS instances, and x86/ARM systems +- Hot configuration reloads reduce the need for service restarts +- Optional **runtime liveness watchdog** can detect a wedged async runtime and capture diagnostics; it can optionally + trigger a restart when explicitly enabled +- Dedicated `/healthcheck` and `/ready` endpoints separate process liveness from actual provider capacity -- Versioned 4 KiB Slotted Page format with checksummed headers, pages, cells, and overflow chains -- Adaptive LZ4 compression and overflow pages for compact variable-size values -- Mmap-backed streaming scans without loading the complete database into memory -- Shared query snapshots reuse mmap mappings, validated pages, and decoded internal routes -- WAL-protected in-place updates with `Immediate` and `Batch` flush policies -- Fully built replacements and compaction results are verified before atomic publication -- Typed startup migration upgrades recognized legacy v1/v2 databases to v3 -- Identity- and generation-bound sorted indexes accelerate playlist-order streaming -- String interning (`Arc`) for playlist entries reduces memory footprint -- Page-local updates, tombstone reuse, free-page reuse, and explicit compaction reclaim disk space +**Benefit:** Tuliprox can stay in the background as infrastructure instead of becoming another heavyweight service stack +you constantly have to maintain. -### 3. Input Sources β€” Bring Everything Together +### 2. Bring All Your Media Sources Together -- **Xtream Codes**: Import Live, VOD, Series, Catchup and EPG metadata through the native provider API. Alias pools and - CSV batch inputs make large account sets manageable. -- **M3U/M3U8**: Load remote or local playlists with custom headers, credentials and EPG sources. CSV batch inputs cover - providers that expose multiple playlist URLs. -- **Stalker & Ministra**: Process complete portal catalogs with MAC, credentials or combined authentication, configurable - MAG profiles, reliable `create_link` playback resolution, resumable refreshes and per-alias CSV configuration. -- **Local Media Library**: Scan movies and series directly from disk, enrich them with NFO or TMDB metadata and publish - them through the same M3U, Xtream and STRM outputs as provider content. -- **Plex, Emby & Jellyfin**: Import selected movie and TV libraries directly from an existing media server, refresh them - manually or on a schedule and resolve protected playback and artwork through Tuliprox. -- **Staged Sources**: Overlay a prepared Live, VOD or Series catalog onto another provider while keeping the original - provider responsible for stream delivery. +Use different upstream technologies without forcing every downstream player to understand them. -### 4. Four Output Formats β€” One Tool to Rule Them All +- **Xtream Codes** β€” Live, VOD, Series, Catchup, EPG metadata, provider aliases, and CSV batch accounts +- **M3U/M3U8** β€” remote or local playlists with headers, credentials, EPG sources, and CSV batch inputs +- **Stalker & Ministra** β€” full portal catalogs with MAC, credential or combined authentication, configurable MAG profiles, + `create_link` playback resolution, resumable refreshes, and per-alias configuration +- **Plex, Emby & Jellyfin** β€” import selected movie and TV libraries and resolve protected playback and artwork through Tuliprox +- **Local Media Library** β€” scan movies and series from disk and enrich them with NFO or TMDB metadata +- **Staged Sources** β€” overlay a prepared Live, VOD, or Series catalog while keeping the original provider responsible for + actual stream delivery -| Format | Description | -|----------------------|---------------------------------------------------------------------------| -| **M3U/M3U8** | For all IPTV players (VLC, Tivimate, iMPlayer, etc.) | -| **Xtream Codes API** | Full Xtream API with Live, VOD, Series, Catchup, EPG | -| **HDHomeRun** | Emulation for Plex, Jellyfin, Emby β€” auto-discovery via SSDP | -| **STRM** | Kodi/Jellyfin/Emby compatible with multi-version support and quality tags | +Tuliprox can also stream large provider catalogs incrementally instead of requiring the complete catalog to be buffered in +memory before processing starts. -Generate all four formats simultaneously from the same source β€” one setup, every platform covered. +**Benefit:** change providers, combine sources, or migrate clients without rebuilding your whole media setup around one vendor-specific API. -### 5. Multi-Tenant IPTV Edge Gateway +### 3. One Catalog, Four Output Formats -- **Advertised Server Profiles**: Define multiple LAN or public reverse-proxy profiles with independent protocol, host, - port, path, timezone and Xtream welcome message, then assign one profile to each user. -- **Target-Based User Access**: Bind every account to a curated target and expose it through M3U or the Xtream Codes API -- **Per-User Delivery Policies**: Choose reverse proxy or redirect behavior, Live/VOD/Series access, category selection, - connection limits and priorities independently for every account -- **Multiple Deployment Profiles**: Advertise LAN and public HTTPS endpoints without duplicating playlists or provider - configuration -- **Flexible User Storage**: Keep accounts in versionable YAML or migrate them to the integrated user database and - manage them from the Web UI -- **Hot-Reloaded Gateway Configuration**: Apply server, user and access-policy changes without restarting the service +| Format | Best for | +|----------------------|----------------------------------------------------------------------------------------| +| **M3U/M3U8** | IPTV players such as VLC, TiviMate, iMPlayer, and similar clients | +| **Xtream Codes API** | Clients expecting Live, VOD, Series, Catchup, EPG, categories, and account information | +| **HDHomeRun** | Plex, Jellyfin, Emby, and compatible tuner-based integrations with SSDP discovery | +| **STRM** | Kodi, Jellyfin, and Emby libraries with multi-version support and quality tags | -### 6. Reverse Proxy, Shared HLS & Stream Management β€” Enterprise-Grade +Generate several output types from the same processed source at the same time. -- **Reverse Proxy Mode**: Streams are proxied through Tuliprox β€” provider URLs stay invisible to end users -- **Redirect Mode**: Lightweight redirection for resource-efficient operation -- **Shared MPEG-TS Streams**: One provider stream shared across multiple users β€” saves valuable provider slots -- **Shared HLS Sessions**: Multiple viewers reuse one server-side HLS session while retaining individual access leases, - user limits and accounting -- **HLS Segment Cache & Prefetch**: Fetch manifests, segments and initialization maps once, serve them locally and - prefetch upcoming segments within configurable disk and concurrency budgets -- **HLS Recovery Tools**: Optional manifest recovery, bounded segment repair and corrupt-segment diagnostics for - difficult upstream streams -- **HLS Session Management**: Short-lived provider reservations for HLS/Catchup without blocking real slots -- **User Connection Priority**: Higher-priority users evict lower-priority connections when all provider slots are full -- **Soft Connections & Soft Priority**: Users can temporarily exceed their normal slot limit with preemptible soft slots; - a dedicated `soft_priority` applies only while a connection is on a soft slot and automatically switches back to the normal - priority when a regular slot becomes free again -- **Grace Period**: Configurable transition window during connection handovers β€” no abrupt drops -- **Bandwidth Throttling**: With flexible units (KB/s, MB/s, kbps, mbps) -- **Per-Stream Metrics**: Bandwidth and transferred bytes per stream in the Web UI (opt-in) -- **Stream History**: Optional persisted stream lifecycle telemetry for connects, disconnects, preemptions, and startup failures -- **QoS Aggregation**: Optional background reliability snapshots built from stream history for long-term stream quality analysis -- **Custom Fallback Videos**: User-defined video files for channel unavailable, connections exhausted, account expired, - etc. -- **Channel-Switch Friendly Reservations**: Instant takeover on channel switch β€” no TTL wait -- **Custom Stream Response Timeout**: Auto-stop fallback streams after configurable duration -- **Buffer Reuse**: Reusable serialization buffers minimize heap allocations during streaming -- **Universal Catchup**: Serve provider catchup/archive as plain M3U with `.ts` segments proxied through Tuliprox β€” - any player, any timeline, no vendor lock-in -- **Smart Background Work**: Skip metadata resolution and probing for entries you don't need β€” save CPU and - provider slots on huge providers that only need partial coverage +**Benefit:** the player no longer dictates how your upstream sources must be organized. -### 7. Provider Connectivity, Failover & VPN Routing +### 4. Powerful Processing Pipeline β€” Make the Catalog Yours -- **Provider URL Failover**: Automatic rotation on errors (5xx, timeout) β€” seamless switching, no viewer disruption -- **Configurable Provider URL Start Policy**: Per provider, choose whether new requests resume from the last working URL or - always restart from the first URL -- **`provider://` URL Scheme**: Reference providers by name β€” Tuliprox resolves to the active URL automatically -- **DNS-Aware Connection Routing**: Provider DNS resolved asynchronously and cached -- **Resolved DNS Persistence**: Resolved IPs persisted separately β€” no source config overwrite during hot reloads -- **Outgoing HTTP/HTTPS/SOCKS5 Proxy**: Route playlist downloads, metadata requests, probing and proxied streams through - a corporate proxy, VPN gateway or Gluetun container -- **Public IP Verification**: Display the effective public IPv4/IPv6 in the Web UI to confirm that VPN and proxy routing - are working as intended -- **Provider Aliases**: Manage multiple accounts from the same provider with different credentials -- **Batch Input**: Xtream, M3U and Stalker batch inputs via CSV files for mass provider management -- **Staged Cluster Source Routing**: Per-cluster decision whether Live/VOD/Series comes from staged input, main input, - or is skipped entirely +Tuliprox can transform provider data into the structure you actually want to use. -### 8. Multi-Source Merging & Advanced Processing Pipeline +- Merge multiple inputs into a single target +- **Filter Engine** with complex boolean expressions, for example: -- Merge multiple input sources (M3U, Xtream, Stalker, Plex, Emby, Jellyfin, Local Library) into a single target -- **Filter Engine**: Complex boolean expressions β€” `(Group ~ "^DE.*") AND NOT (Name ~ ".*XXX.*")` -- **Mapper DSL**: A custom Domain-Specific Language for powerful transformations - - Regex-based renaming with capture groups and backreferences - - Variables, if/else blocks, loops (`for_each`) - - Built-in functions: `replace`, `pad`, `format`, `first`, `capitalize`, `lowercase`, `uppercase`, `template` - - Counters with padding (e.g. `001`, `002`) - - Transform operations directly in mappings -- **Template System**: Centralized, reusable pattern collection - - Global templates shared across sources and mappings - - Inline templates for backward compatibility - - List templates for sequences - - Hot-reload on template changes -- **Sort Engine**: - - Regex sequences for groups and channels - - `order: none` to preserve source order - - Filter-based sorting β€” sort only specific entries - - Named capture groups for multi-level sorting (`c1`, `c2`, `c3`) -- **Accent-Independent Matching**: `match_as_ascii` β€” "Cinema" matches "CinΓ©ma" -- **Deunicoding**: On-the-fly Unicode normalization in filters and value comparisons -- **Output Filters**: Apply filters to the final playlist state after all transformations -- **Favorites System**: Explicit `add_favourite(group_name)` script function for bouquet management -- **Auto-Curated Trakt Bouquets**: Drop in Trakt Charts and serve "Trending Now" and "Most Popular" as live Xtream - categories β€” your playlist refreshes itself with what people are actually watching, no manual curation + ```text + (Group ~ "^DE.*") AND NOT (Name ~ ".*XXX.*") + ``` -### 9. Local Media Library β€” Integrate Your Own Movies & Series +- Target filters can run during normal processing or at the final **persist** stage after EPG processing, mapping, merging, + deduplication, sorting, numbering, and counters +- **Mapper DSL** for advanced transformations: + - regex renaming with capture groups and backreferences + - variables, `if`/`else`, and `for_each` + - functions such as `replace`, `pad`, `format`, `first`, `capitalize`, `lowercase`, `uppercase`, and `template` + - counters with padding such as `001`, `002`, `003` + - read-only metadata fields such as `@Input` and `@Type` + - mapping stages including normal processing and `after_epg` +- **Reusable templates** β€” global, inline, list-based, and hot-reloaded +- **Sort Engine** β€” regex sequences, source-order preservation, filtered sorting, and named capture groups +- Accent-independent matching with `match_as_ascii` +- Unicode normalization for filters and value comparisons +- Output filters for the final playlist state +- Favorites/bouquet management through `add_favourite(group_name)` +- Target-specific bouquet whitelist/blacklist editing directly from the Web UI +- Trakt-based auto-curated bouquets such as trending and popular content -- Recursive directory scanning for local video files -- Automatic classification (Movie vs. Series) -- Multi-source metadata: NFO files, TMDB API, filename parsing -- Incremental scanning β€” only new or changed files are processed -- Integration into Xtream API and M3U playlists β€” local content served like IPTV channels -- Metadata formats including NFO support -- Scheduled library scans β€” automatic updates via cron -- Episode backgrounds with direct TMDB image URLs -- Virtual ID management for stable assignment +**Benefit:** providers supply the raw catalog; Tuliprox decides what your users actually see and how it is organized. -### 10. Metadata Resolution & Stream Probing +### 5. Protect Provider Slots and Improve Stream Delivery -- **Background Metadata Queue**: Metadata resolution and stream analysis run in the background when provider - connections - are idle β€” prevents "No Connections" errors for active users -- **Metadata Fairness**: Configurable ratio between resolve and probe tasks β€” no probe starvation -- **FFprobe Integration**: Automatic detection of codec, resolution, HDR (HDR10/HLG/Dolby Vision), audio channels -- **FFprobe respects provider limits**: If no slot is available, the item is skipped β€” zero risk of provider bans -- **TMDB Integration**: Automatic lookup of missing TMDB IDs and release dates -- **TMDB No-Match Cooldown**: Prevents infinite loops on items with no TMDB match -- **Quality Tagging**: Automatic STRM tags like `[2160p 4K HEVC HDR TrueHD 7.1]` -- **Flat Grouping**: Multi-version merge (e.g. 4K + 1080p) in a single folder β€” compatible with Jellyfin/Emby - multi-version feature -- **Probe Priority**: Configurable priority for probe tasks β€” provider slots stay free for real users -- **Metadata Retry State**: Persistent retry/cooldown state per item in a dedicated database -- **No-Change Cache**: Deduplication cache prevents unnecessary re-resolution of unchanged items -- **Live Stream Probing**: Periodic re-probing of live streams with configurable interval +Tuliprox is not just a metadata processor β€” it actively manages stream delivery. -### 11. Role-Based Access Control (RBAC) β€” Enterprise-Grade Security +- **Reverse Proxy Mode** β€” hide provider URLs and keep client traffic behind Tuliprox +- **Redirect Mode** β€” avoid proxy overhead where direct delivery is preferred +- **Shared MPEG-TS Streams** β€” several viewers can reuse one upstream provider stream +- **Shared HLS Sessions** β€” reuse one server-side HLS session while retaining individual user leases and accounting +- **HLS Segment Cache & Prefetch** β€” fetch manifests, initialization maps, and segments once and serve them locally +- **HLS Recovery** β€” optional manifest recovery, bounded segment repair, stale-origin recovery, and diagnostics for difficult upstream streams +- **Universal Catchup** β€” expose provider archive/catchup through regular M3U-compatible `.ts` URLs +- **Bandwidth Throttling** with flexible units +- **Channel-switch friendly reservations** to avoid unnecessary slot waits +- **Fallback videos** for unavailable channels, exhausted capacity, expired accounts, and similar conditions +- Configurable fallback error mode for reverse proxies that should handle failures themselves -- **14 permissions** across 7 domains (config, source, user, playlist, library, system, epg) -- Each permission with independent `.read` and `.write` grants -- Custom groups via `groups.txt` β€” define your own roles (e.g. `viewer`, `source_manager`) -- User-to-group assignment β€” one user can belong to multiple groups -- Union-based permission resolution β€” group permissions stack additively -- Compact bitmask (`u16`) in JWT β€” zero file I/O per request, single-instruction bitwise checks -- Password-version tracking in JWT β€” automatic token invalidation on password change -- RBAC admin panel in the Web UI with tabbed user/group management and permission checkbox grid -- Built-in `admin` group β€” always full permissions, cannot be deleted -- Backward compatible β€” existing `user.txt` files work without changes +#### Connection admission and priority -### 12. Web UI β€” Full Control in the Browser +- Normal and soft user connection limits +- Dedicated soft priority for temporary overflow connections +- Provider and user priority handling +- Ordered admission strategies for deciding what happens when limits are reached: + - evict oldest/newest connections from the same user + - optionally scope eviction to the same IP + - instant or held grace-period strategies +- Anti-ping-pong protection for aggressive reconnect loops +- Session-aware handling for HLS/catchup and socket-bound admission for regular TS/VOD/local playback -- **Dashboard**: System status, active streams, CPU usage, provider connections in real-time via WebSocket -- **Source Editor**: Dedicated forms for M3U, Xtream, Stalker, Plex, Emby, Jellyfin and local-library inputs, with - drag & drop, block selection and batch mode; target forms also provide a full-size bouquet editor with whitelist and - blacklist group selection -- **Playlist Explorer**: Tree and gallery view for channels with EPG timeline and search β€” text or regex, optionally - scoped to specific fields (group, title, name, url) -- **Download & Recording Manager**: Provider-aware VOD downloads and live recordings with retries, fairness, and RBAC-controlled actions -- **Config Editor**: Direct editing of config.yml, source.yml, mapping.yml in the browser -- **User Management**: API users with category selection, priority, soft-priority, normal/soft connection limits, auto-generated credentials, - and reusable **plans** (capability tiers with cluster access, connection limits, and admin-enforced content filters) -- **Network Access Policy UI**: API-user network restrictions can be configured with CIDR and GeoIP country rules, including - the global GeoIP-unavailable `deny`/`allow` policy. -- **RBAC Admin Panel**: Tabbed user/group management, permission checkbox grid, write-without-read warnings -- **Stream Table**: Real-time stream monitoring with copy-to-clipboard, bandwidth metrics, episode titles -- **EPG View**: Timeline with channels, now-line, program details -- **Messaging Config View**: Discord, Telegram, Pushover, REST webhook configuration with template editor -- **Multiple Themes**: Dark/bright and additional themes available -- **GeoIP Country Flags**: Country flags displayed when GeoIP is active -- **Playlist User Login**: Playlist users can select their own groups/bouquets -- **Resource Proxy**: Channel logos and images loaded via authenticated same-origin endpoints β€” HTTP upstream assets - render behind HTTPS frontends -- **Mobile-friendly**: Responsive design for all screen sizes +**Benefit:** limited provider capacity can be shared intentionally instead of being consumed unpredictably by clients, +reconnects, probes, and background jobs. -**UI/UX Highlights:** +### 6. Provider Resilience, Failover, and Safer Refreshes -- **Status Health Banner**: Single green/amber/red health indicator in the header aggregates realtime connection, - backend, and provider capacity with hover breakdown and click-through to the Stats view -- **Live Metric Sparklines**: Stats cards show interactive time-series sparklines for CPU, memory, network, active - users, and active user connections β€” hover shows cursor + tooltip, system metrics sampled every 2 s for responsive - charts -- **Bookmarkable Views (Deep Linking)**: The active view is reflected in the URL hash (`#stats`, - `#source_editor`) so views can be bookmarked, shared, and navigated directly; browser back/forward works as - expected -- **At a glance**: Single green/amber/red health banner in the header aggregates every signal that matters - (realtime connection, backend, provider capacity), with hover breakdown and click-through to detailed Stats. - Live metric sparklines for CPU, memory, network, active users and connections β€” hover for cursor + tooltip, - sampled every 2 s so the charts never feel stale. -- **A UI that feels right**: Micro-interactions (cards lift, buttons ripple, chevrons rotate), animated - theme transitions, debounced filter input that stays responsive on huge expressions, shift+click range - selection in the category editor, guided empty states that tell you what to do next, paged tables with - localized empty messages. Every effect honors `prefers-reduced-motion: reduce`. -- **Works the way you work**: Bookmarkable views via URL hash (`#stats`, `#source_editor`) so you can share - links and the browser back/forward buttons just work. Persisted sidebar state and table page size - survive reloads. Sessions auto-logout cleanly when the JWT expires β€” no mysterious 401s. -- **For everyone**: UI languages with full RTL support (drop a file in `assets/i18n` and a new language is live β€” no rebuild), - a recoverable error boundary that keeps one misbehaving view from taking the app down, and confirmation dialogs before any - destructive download/recording action. +Upstream providers are not always reliable. Tuliprox is designed to preserve usable service when they fail partially or temporarily. -### 13. EPG (Electronic Program Guide) +- Automatic provider URL failover on timeouts and server errors +- Choose whether a provider resumes from the last working URL or restarts from the first URL on future requests +- `provider://` references resolve to the currently active provider URL +- Asynchronous DNS resolution and caching +- Persisted resolved IPs without overwriting source configuration +- HTTP, HTTPS, and SOCKS5 outbound proxy support for VPN gateways, corporate proxies, or Gluetun +- Effective public IPv4/IPv6 display in the Web UI to verify routing +- Provider aliases and CSV batch configuration for larger account pools +- **Per-cluster update quality guards** for Xtream, Stalker, and M3U inputs can reject suspicious Live/VOD/Series refreshes + and retain the last accepted data +- Quality rejections are reported separately from technical failures +- Independent inputs can update in parallel while optional sequential groups protect providers that must not be refreshed concurrently +- Automatic Xtream account-expiration refresh with throttling designed to reduce unnecessary provider requests +- Stalker capability knowledge is reused during a running client session to avoid repeatedly probing known-failing endpoint variants -- **Multi-Source EPG**: Multiple EPG sources with priorities β€” best coverage through combination -- **ICS Calendar Sources**: Convert iCalendar events into XMLTV programmes, assign them through Smart Match and - optionally fill schedule gaps with configurable dummy blocks -- **Auto-EPG**: Automatic EPG URL generation from providers -- **Smart Match**: Fuzzy matching with configurable threshold for automatic channel-to-EPG assignment -- **XMLTV Timeshift**: Full timezone support (`Europe/Paris`, `America/New_York`, `-2:30`, `+0:15`, etc.) with automatic - DST handling -- **EPG Memory Cache**: In-memory cache for fast Web UI and short-EPG lookups β€” reduces disk access -- **Logo Override**: EPG logos can override channel logos -- **Async Processing**: EPG streamed and processed asynchronously β€” minimal memory overhead even with large guides -- **EPG Icon Proxy**: HTTP upstream assets rendered through HTTPS frontend -- **Strip & Normalize**: Configurable terms and regex for better channel matching -- **EPG Title Synchronization**: Automatic sync after playlist updates +**Benefit:** a broken refresh or failing alias does not have to destroy an otherwise usable catalog. -### 14. Notifications & Monitoring +### 7. Metadata, EPG, and Stream Intelligence -- **Telegram**: Bot notifications with markdown support and thread support (`chat-id:thread-id`) -- **Discord**: Webhook notifications with Handlebars templates -- **Pushover**: Push notifications -- **REST Webhooks**: Custom HTTP methods, headers, Handlebars templating -- **Watch Notifications**: Real-time alerts on group changes in playlists -- **Processing Stats**: Automatic notification after playlist updates with statistics and processing duration -- **Per-Message Templates**: Individual Handlebars templates per message type (Info, Stats, Error, Watch) and channel -- **Template Loading**: Templates from files or HTTP/HTTPS URIs with automatic discovery -- **Typed Messaging Pipeline**: Strictly typed pipeline instead of raw JSON strings β€” robust and maintainable -- **Never Fill Up Unnoticed**: Disk-usage alerts through your existing Telegram/Discord/Pushover channels - β€” opt-in, configurable thresholds, hourly re-arm so a stuck-at-95% disk keeps nagging until you actually - fix it +Tuliprox can improve incomplete provider data and make large catalogs easier to consume. -### 15. Scheduling & Automation +- Multi-source EPG with priorities +- Automatic provider EPG URL generation +- ICS/iCalendar sources converted into XMLTV programmes +- Smart fuzzy matching between playlist channels and EPG channels +- Configurable XMLTV timeshift with timezone and DST support +- Configurable normalization and cleanup for better matching +- Optional clearing of invalid EPG IDs without removing the playlist entry itself +- EPG logo override and authenticated image proxying +- In-memory EPG cache for fast UI and API access +- Asynchronous EPG processing for large guides +- Automatic EPG title synchronization after playlist updates +- Extended programme metadata where available -- **Cron-based scheduler**: Multiple schedules with optional target selection -- **Scheduled library scans**: Automatic local library scans alongside playlist updates -- **Hot config reload**: Configuration changes detected and applied automatically β€” no restart needed -- **Config file watcher**: Monitors config.yml, source.yml, mapping.yml, api-proxy.yml, template files -- **Auto-update on boot**: Optional playlist update on startup -- **Staged inputs**: Side-loading β€” load metadata from staged input, serve streams from provider -- **Panel API integration**: Auto-renew expired provider accounts or provision new ones to maintain minimum valid - accounts -- **Playlist caching**: Configurable cache duration for provider playlists (`60s`, `5m`, `12h`, `1d`) +#### Metadata resolution and probing -### 16. Management REST API β€” Automate Everything +- Background metadata resolution and stream probing when provider capacity is available +- Configurable fairness between metadata resolution and probing +- FFprobe-based codec, resolution, HDR, and audio-channel detection +- Provider-slot-aware probing β€” skip work instead of stealing the last available connection +- TMDB lookup for missing IDs and release information +- Persistent retry/cooldown state for failed metadata operations +- No-change cache to avoid repeating work for unchanged items +- Periodic live-stream probing +- Automatic quality tags such as `[2160p 4K HEVC HDR TrueHD 7.1]` +- Multi-version grouping for Jellyfin/Emby-style libraries -- **Configuration Management**: Read and update application, source and API-gateway configuration remotely -- **Processing Control**: Trigger playlist refreshes, target-scoped updates, library scans and GeoIP database updates -- **Playlist Inspection**: Preview Live, VOD and Series content before publishing it to users -- **Transfer Automation**: Queue, pause, resume, retry and cancel downloads or live recordings from external tools -- **Operational Visibility**: Query status, active streams, stream history, QoS snapshots and system information -- **Script-Friendly Formats**: Use JSON for broad compatibility or CBOR on supported endpoints for compact responses +**Benefit:** clients receive a cleaner, richer catalog without each player having to solve metadata and EPG inconsistencies on its own. -### 17. Complete Xtream Codes API Implementation +### 8. Web UI β€” Operate Tuliprox Without Living in YAML -- Player API (Live, VOD, Series streams) -- Category listings with icons -- VOD Info & Series Info with episodes and metadata -- Catchup / Timeshift API with session tracking -- XMLTV / EPG API -- Panel API for account management -- POST and GET request support -- Series/Catchup lookup with virtual ID support -- Bandwidth and connection info in user info response -- Custom server message support -- Multi-server configuration with different protocols and ports +Tuliprox can be configured and monitored directly from the browser. -### 18. Security +- **Dashboard** with backend status, active streams, system metrics, and provider capacity +- **Source Editor** for M3U, Xtream, Stalker, Plex, Emby, Jellyfin, local-library, and target configuration +- Drag & drop, block selection, batch editing, and target bouquet whitelist/blacklist editing +- **Input-focused Update view** with per-input actions, affected-target selection, status, last update time, and + capability-aware Update/Refresh/Force Update/Rescan actions +- **Update Details** explain where data came from, which quality guards were applied, processing order, target results, + and technical versus quality failures +- Reload-safe update/cluster status so a browser refresh does not erase the operational picture +- **Playlist Explorer** with tree/gallery views, EPG timeline, text/regex search, and field-scoped search +- **Download & Recording Manager** with retries, queue controls, fairness, and RBAC-protected actions +- **User Management** including plans, connection limits, priorities, category access, output clusters, and network policies +- **RBAC Administration** for users, groups, and permission grants +- **Stream View** with bandwidth, transferred bytes, player information, comments, and EPG details +- **Stream History & QoS** views for reliability analysis +- **Messaging Configuration** with channel routing, templates, and test/preview support +- Configurable landing page and stream-field visibility +- Multiple themes including a color-vision-deficiency-friendly option +- Runtime-discovered UI languages with RTL support +- Bookmarkable views via URL hash and working browser back/forward navigation +- Responsive design for desktop and mobile -- **Argon2 password hashing**: Industry standard for password storage -- **JWT authentication**: Compact bitmask encoding with password-version tracking for automatic token invalidation -- **Rate limiting**: Per-IP rate limiting with configurable burst and period -- **Content Security Policy**: Configurable CSP headers -- **API User Network Access Restrictions**: Per-user `network_access` rules can restrict API proxy accounts by IPv4/IPv6 - CIDR ranges and/or GeoIP country codes. Matching any configured CIDR or country allows the request; otherwise the - request is denied before it is forwarded upstream. -- **GeoIP-Unavailable Policy**: Country-based network restrictions default to `deny` when GeoIP is disabled, missing, or not - loaded. Operators can explicitly accept that risk with `reverse_proxy.geoip.unavailable_policy: allow`; CIDR-only misses, - unknown countries, and country mismatches still deny. -- **SSL/TLS support**: Configurable including `accept_insecure_ssl_certificates` option -- **Header stripping**: Configurable removal of referer, Cloudflare, and X-headers -- **Rewrite secret**: Mandatory secret for stable resource URLs in reverse proxy mode +#### UX and accessibility -### 19. Operations & Deployment +- Health banner with green/amber/red capacity and backend state +- Live metric sparklines for CPU, memory, network, active users, and connections +- Persisted sidebar and table preferences +- Debounced filter editing for large expressions +- Guided empty states instead of unexplained blank views +- Recoverable per-view error boundaries +- Keyboard-friendly dialogs and menus +- Screen-reader improvements and descriptive image alt text +- Reduced-motion support for animations and micro-interactions +- Confirmation before destructive download/recording operations -- **Docker**: Alpine and Scratch images β€” minimal image size -- **Docker Compose templates**: traefik, crowdsec, gluetun/socks5, iptv-org/epg templates ready to use -- **One-Click Dev Setup**: Open in GitHub Codespaces or VS Code Dev Containers and start hacking β€” Rust - toolchain, WASM targets, and every build tool pre-configured -- **Zero-downtime config reload**: `ArcSwap` for atomic configuration swaps without interruption -- **Disk-based processing**: Playlist processing from disk instead of RAM β€” massively reduced memory consumption -- **CLI mode**: One-shot processing without a server β€” ideal for scripting and CI/CD -- **Server mode**: Long-running HTTP server with background tasks -- **Healthcheck endpoints**: - - `/healthcheck` β€” liveness probe (HTTP 200 while the process is running). Used by Docker via the `--healthcheck` CLI flag. - - `/ready` β€” readiness probe (HTTP 200 when at least one input group has spare connection capacity; 503 when every input group is exhausted). - - `/api/v1/status` β€” detailed status payload (counts, version, build time) β€” not intended for Docker. -- **SSDP discovery**: HDHomeRun auto-discovery via SSDP and proprietary UDP protocol (port 65001) -- **Database viewer**: CLI flags to inspect internal databases -- **Environment variables**: `${env:VAR}` interpolation in all config files -- **Default User-Agent**: Configurable default user-agent for all outgoing requests +**Benefit:** advanced functionality stays available to power users without forcing routine administration onto the command line. -### Configuration & Secrets +### 9. Multi-User Gateway, Plans, and Access Control + +Expose one Tuliprox instance to multiple users without giving everyone identical access. + +- Assign each account to a processed target +- Advertise different LAN/public server profiles per user +- Choose reverse-proxy or redirect behavior per account +- Restrict Live, VOD, and Series independently through per-user output clusters +- Select allowed categories/bouquets +- Configure normal and soft connection limits +- Configure connection priority and soft priority +- Reusable **plans** for capability tiers, cluster access, connection limits, and enforced filters +- Store users in versionable configuration or the integrated user database +- Apply user and gateway configuration changes without restarting the service + +#### RBAC + +- 14 permissions across 7 domains +- Independent `.read` and `.write` grants +- Custom groups and multi-group membership +- Additive permission resolution +- Compact permission bitmask in JWTs +- Built-in `admin` group with protected full access + +**Benefit:** one backend can serve different devices, households, environments, or access tiers without duplicating the provider and processing configuration. + +### 10. Security Built Into the Control Plane + +- Argon2 password hashing +- JWT authentication with password-version tracking +- Sign-in throttling by client address and username +- Persistent token revocation for one user or all users +- Auth audit events for successful, failed, throttled, and permission-denied activity +- Per-IP request rate limiting +- Configurable Content Security Policy +- Per-user IPv4/IPv6 CIDR restrictions +- Per-user GeoIP country restrictions with secure deny behavior when GeoIP data is unavailable by default +- Configurable SSL/TLS behavior +- Header stripping for selected upstream/request headers +- Mandatory rewrite secret for stable resource URLs in reverse-proxy mode +- Sensitive values masked in configuration API responses + +**Benefit:** Tuliprox can be exposed as a real multi-user gateway without treating authentication and access policies as an afterthought. + +### 11. Event-Based Notifications and Monitoring + +Tuliprox uses a shared event backbone so operational events can feed the Web UI, notifications, diagnostics, and future integrations consistently. + +Supported notification channels include: + +- Telegram +- Discord +- Pushover +- REST webhooks +- ntfy +- Gotify +- Slack +- Local command execution with structured event JSON on stdin + +Notification capabilities include: + +- Event subscriptions using dotted event IDs and glob patterns +- Per-channel minimum severity +- Per-channel subscriptions and routing +- Quiet hours that defer notifications instead of silently dropping them +- Deduplication windows and per-hour rate caps +- Durable delivery with retry handling for transient failures +- Dead-letter tracking for notifications that permanently fail +- Provider `Retry-After` support where applicable +- HMAC-SHA256 signing for REST webhooks +- Per-channel Handlebars templates +- Preview/test endpoint so templates can be rendered without sending them +- Events for provider failures, exhausted pools, priority fallback, user connection denial, auth activity, config reload + failures, metadata failures, scheduled-task failures, recording lifecycle, disk alerts, and more +- Event statistics and a recent-event ring for troubleshooting +- Disk usage alerts with warning/critical thresholds and configurable repeat intervals + +**Benefit:** failures that would otherwise live only in a log file can reach the operator through the channel that is actually monitored. + +### 12. Stream History and QoS Visibility + +Optional stream telemetry can turn individual playback failures into longer-term reliability information. + +- Persist stream connects, disconnects, preemptions, session expiry, and startup failures +- Classify failure stages and provider error classes +- Track stable stream identity across runs +- Track shared-stream participation +- Aggregate rolling reliability snapshots in the background +- Maintain 24-hour, 7-day, and 30-day QoS windows +- View QoS summaries/details in the Web UI or query them through the API +- Compact QoS storage periodically to reclaim expired data + +**Benefit:** troubleshooting moves from β€œthis channel sometimes fails” to data that can show where and how often it fails. + +### 13. Digital Video Recorder (DVR) + +Turn existing Tuliprox sources into a managed recording library without adding another DVR server and database stack. + +- Record live channels directly through Tuliprox +- Scheduled and recurring recording rules +- Conflict preview before recordings start +- Provider-aware scheduling that respects available connection capacity +- Reliable queue operations: pause, resume, retry, edit, and cancel +- Crash-safe startup reconciliation +- Automatic retention by age, count, and disk-space thresholds +- Recording quota management +- Configurable recording container format +- Durable lifecycle notifications with per-channel retry handling +- Real-time Web UI updates for recordings and rules +- Protected media, thumbnail, and subtitle paths with filesystem containment checks +- Hot-reloadable DVR configuration +- Built-in recording subsystem health endpoint +- `dvr_doctor` diagnostics for support and troubleshooting + +**Benefit:** recording uses the same sources, permissions, provider limits, monitoring, and management UI you already configured for playback. + +### 14. Embedded Storage β€” No External Database Required + +Tuliprox includes its own storage engine for playlist and runtime data. + +- Versioned 4 KiB slotted-page format +- Checksummed headers, pages, cells, and overflow chains +- Adaptive LZ4 compression +- Mmap-backed streaming scans instead of loading the complete database into RAM +- Shared query snapshots for reused mappings and validated routes +- WAL-protected in-place updates with immediate and batch flush policies +- Atomic publication of verified replacements and compaction results +- Startup migration for recognized legacy database versions +- Sorted indexes for playlist-order streaming +- String interning for repeated playlist values +- Tombstone and free-page reuse plus explicit compaction + +**Benefit:** you get durable local state without deploying and maintaining PostgreSQL, MySQL, MongoDB, or Redis just to run Tuliprox. + +### 15. Scheduling and Automation + +- Cron-based scheduling with optional target selection +- Scheduled local-library scans +- Optional playlist update on startup +- Configuration file watching and hot reload +- Playlist/provider caching with configurable duration +- Staged-input processing +- Panel API integration for provider account provisioning/renewal workflows +- Dependency-aware parallel input processing +- Management REST API for configuration, processing, transfers, recordings, status, stream history, QoS, and system information +- JSON and CBOR responses on supported endpoints +- CLI one-shot processing for scripts and CI/CD + +**Benefit:** common maintenance tasks can run unattended, while the same operations remain available through the UI, API, +or CLI when manual control is needed. + +### 16. Complete Xtream-Compatible Delivery + +- Player API for Live, VOD, and Series +- Categories and icons +- VOD and Series detail endpoints +- Catchup/timeshift support with session tracking +- XMLTV/EPG endpoints +- Panel API integration +- GET and POST request handling +- Series/Catchup lookup with virtual IDs +- Connection and bandwidth information in user responses +- Custom server messages +- Multiple advertised server profiles with different protocols, hosts, ports, and paths + +### 17. Local Media Library + +- Recursive directory scanning +- Automatic Movie/Series classification +- NFO, TMDB, and filename-based metadata resolution +- Incremental scans for new or changed files +- Scheduled rescans +- Stable virtual IDs +- Episode artwork/background integration +- Publish local content through the same Xtream, M3U, and STRM outputs as provider content +- Rebuild selected targets automatically after successful library rescans + +**Benefit:** local files and provider content can live in one catalog instead of separate client libraries and workflows. + +### 18. Operations and Deployment + +- Alpine and Scratch Docker images +- Docker Compose templates for Traefik, CrowdSec, Gluetun/SOCKS5, iptv-org/epg, and Tuliprox +- Zero-downtime configuration swaps with `ArcSwap` +- Disk-based processing for large playlists +- Server and CLI modes +- `/healthcheck` liveness endpoint +- `/ready` capacity-aware readiness endpoint +- `/api/v1/status` detailed status endpoint +- SSDP/HDHomeRun discovery +- Database inspection tooling +- `${env:VAR}` interpolation throughout configuration +- `.env` loading with host/container environment variables taking precedence +- Optional startup runtime-configuration report in YAML or JSON with sensitive values redacted +- Configurable default outbound User-Agent +- Dev Container/Codespaces environment with the Rust/WASM tooling needed for development + +## πŸ” Configuration and Secrets Tuliprox ships with demo placeholder configuration only β€” nothing in this repository is a real credential. -- Every config file supports `${env:VAR}` interpolation (`config.yml`, `source.yml`, `api-proxy.yml`, - `mapping.yml`, `template.yml`), so provider logins, webhooks, API keys and tokens can be injected from the - process environment. No particular host or secret store is assumed. -- `config/user.txt` contains sample Argon2 hashes for the demo accounts `test` / `nobody`. Replace them with - hashes generated on your machine via `tuliprox --genpwd`. -- Pin `web_ui.auth.secret` (64-hex) across restarts so logins survive reboots. -- See [Secrets & Environment Variables](docs/src/configuration/secrets.md) for the full guide and the - pre-publish checklist. +- All major config files support `${env:VAR}` interpolation, including `config.yml`, `source.yml`, `api-proxy.yml`, `mapping.yml`, and `template.yml` +- Tuliprox can load variables from a `.env` file at startup; existing process/Docker environment variables take precedence +- Provider credentials, webhook secrets, API keys, and tokens can therefore stay outside version-controlled configuration +- `config/user.txt` contains sample Argon2 hashes for demo accounts only; replace them with hashes generated locally using `tuliprox --genpwd` +- Pin `web_ui.auth.secret` across restarts so active logins can survive a service restart +- Configuration API responses mask sensitive messaging secrets rather than returning them in clear text +- See [Secrets & Environment Variables](docs/src/configuration/secrets.md) for the full guide and pre-publish checklist -### 20. Digital Video Recorder (DVR) β€” Record, Manage & Keep What Matters +## 🎯 Who Is Tuliprox For? -Turn Tuliprox into your personal IPTV recorder. Record live TV directly from your existing sources, automate recurring recordings, -manage everything from the Web UI, and let Tuliprox handle storage, retries and cleanup in the background. +### IPTV Enthusiasts -- **Live TV Recording**: Record channels directly through Tuliprox without a separate DVR service -- **Scheduled & Recurring Recordings**: Create recording rules for programs you want to capture automatically -- **Smart Conflict Detection**: Preview potential recording conflicts before they happen and identify recordings that may exceed available provider capacity -- **Provider-Aware Scheduling**: Recording jobs respect available provider connections instead of blindly consuming stream slots -- **Reliable Recording Queue**: Queue, pause, resume, retry, edit or cancel recording jobs while Tuliprox keeps the queue consistent -- **Crash-Safe Recovery**: Interrupted operations are automatically reconciled after a restart so the recording library stays consistent -- **Automatic Retention**: Keep storage under control with configurable age, recording-count and disk-space policies -- **Quota Management**: Recording storage can be tracked and limited instead of allowing recordings to consume the entire disk -- **Durable Notifications**: Recording lifecycle events can be delivered through configured notification channels with automatic - retries when individual services are temporarily unavailable -- **Real-Time Web UI Updates**: Recording and recording-rule changes are pushed directly to connected Web UI clients -- **Secure Media Access**: Recorded media, thumbnails and subtitles are served through protected paths with strict filesystem containment -- **Built for 24/7 Operation**: Dedicated background supervisors continuously handle recovery, retention and notifications -- **Hot-Reloadable Configuration**: Change DVR settings without restarting Tuliprox -- **Built-In Health Monitoring**: Inspect DVR subsystem health, background processing and notification queues from the management interface -- **Support-Friendly Diagnostics**: A bundled DVR doctor command collects recording configuration, storage state, quota information and - supervisor health for troubleshooting +- Merge several providers into one catalog +- Normalize inconsistent names, groups, logos, and EPG data +- Keep only the content you actually want +- Use the same cleaned catalog in different players +- Share provider streams and make better use of limited connection slots -**Your IPTV sources become your own managed recording library β€” without adding another database, DVR server or separate management stack.** +### Self-Hosted and Homelab Users -## 🎯 Target Audiences +- Run a complete gateway in one container without an external database stack +- Deploy on Raspberry Pi, NAS, VPS, or a larger home server +- Route upstream traffic through a VPN or SOCKS5 gateway +- Verify the effective public IP directly from the Web UI +- Monitor health, capacity, disk usage, streams, and notifications from one place +- Use Traefik, CrowdSec, Gluetun, and EPG templates as building blocks -### For IPTV Enthusiasts +### Multi-User Setups -- Merge multiple providers into one unified playlist -- Filter, rename, sort channels β€” exactly the way you want -- Automatic EPG assignment with fuzzy matching -- Kodi/Jellyfin/Emby integration via STRM -- Plex integration via HDHomeRun +- Publish several virtual IPTV endpoints from one Tuliprox instance +- Assign different targets, plans, categories, clusters, limits, and priorities per user +- Share provider capacity while retaining per-user accounting and admission control +- Restrict access by CIDR and GeoIP policy +- Use RBAC for administrative access to Tuliprox itself -### For Self-Hosted & Homelab Users +### Developers and Power Users -- Single Docker container β€” no database stack needed -- Runs on Raspberry Pi and tiny VPS instances -- Minimal resource usage thanks to Rust and disk-based processing -- Route all upstream traffic through a VPN or SOCKS5 gateway and verify the public IP from the Web UI -- **Runs 24/7 for months with rock-solid stability and near-zero maintenance** -- Traefik/Crowdsec/Gluetun/IPTV-org-epg templates ready to deploy - -### For Multi-User Operations - -- Publish multiple virtual IPTV endpoints and plans from one Tuliprox instance -- User management with connection limits and priority levels -- RBAC with 14 granular permissions across 7 domains -- Provider slot sharing for live streams -- Custom fallback videos for professional support -- Panel API integration for automated account management - -### For Developers & Power Users - -- Mapper DSL for arbitrary transformations -- Template system for reusability -- Management REST API for configuration, processing, transfers and operational automation +- Mapper DSL for complex transformations +- Reusable templates and staged mapping +- Management REST API +- Event-based automation and signed webhooks - CLI mode for scripting and CI/CD -- Database viewer for debugging and analysis +- Database viewers and diagnostics +- Reproducible Dev Container/Codespaces setup ## πŸ‹ Docker Container Templates @@ -427,15 +507,17 @@ manage everything from the Web UI, and let Tuliprox handle storage, retries and - [crowdsec](docker/container-templates/crowdsec/) template - [gluetun/socks5](docker/container-templates/gluetun/) template - [iptv-org-epg](docker/container-templates/iptv-org/) template -- [tuliprox](docker/container-templates/tuliprox/) (incl. traefik) template +- [tuliprox](docker/container-templates/tuliprox/) (including Traefik) template -`> ./docker/container-templates` +```text +./docker/container-templates +``` -## Quick start +## ⚑ Quick Start -Install with docker the latest image. +Start the latest Docker image: -```docker +```yaml services: tuliprox: container_name: tuliprox @@ -452,19 +534,25 @@ services: restart: unless-stopped ``` -Open the Browser and continue setup. +Then open the Web UI in your browser and continue the setup there. -## Project layout +## 🀝 Community -- `backend/`: backend crates (`app/` contains the main server and processing pipeline) -- `frontend/`: Yew Web UI -- `shared/`: DTOs and shared logic -- `config/`: example configuration -- `docs/`: Markdown source for the project documentation +Questions, ideas, feedback, and contributions are welcome. -## Documentation +[Join the Tuliprox Discord community](https://discord.gg/gkzCmWw9Tf) -The detailed documentation lives in Markdown under `docs/` and is meant to be rendered as a static site. +## 🧱 Project Layout + +- `backend/` β€” backend crates; `app/` contains the main server and processing pipeline +- `frontend/` β€” Yew Web UI +- `shared/` β€” DTOs and shared logic +- `config/` β€” example configuration +- `docs/` β€” Markdown source for the project documentation + +## πŸ“š Documentation + +The detailed documentation lives in Markdown under `docs/` and is designed to be rendered as a static site. - Docs source: [`docs/src/index.md`](docs/src/index.md) - Build static docs: `make docs` @@ -486,7 +574,7 @@ Main entry points: - [Operations & Debugging](docs/src/operations-debugging.md) - [Troubleshooting & Resilience](docs/src/troubleshooting.md) -## Documentation strategy +### Documentation Strategy The recommended format is: @@ -494,9 +582,13 @@ The recommended format is: - generated as static HTML - shipped together with the frontend/web root -For this repository, `mdBook` is the best fit: +For this repository, `mdBook` is a natural fit: -- Markdown stays easy to edit in Git +- Markdown stays easy to review and edit in Git - static HTML output is simple to host -- it fits a Rust project better than a Node-heavy doc stack -- navigation and search come out of the box +- it fits the Rust ecosystem well +- navigation and search are available out of the box + +## πŸ“„ License + +See [`LICENSE`](https://github.com/euzu/tuliprox/blob/develop/LICENSE). diff --git a/backend/app/Cargo.toml b/backend/app/Cargo.toml index a0a18c42a..a1349f3dd 100644 --- a/backend/app/Cargo.toml +++ b/backend/app/Cargo.toml @@ -3,6 +3,7 @@ name = "tuliprox" version.workspace = true edition.workspace = true rust-version.workspace = true +default-run = "tuliprox" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -62,9 +63,6 @@ blake3.workspace = true bytes.workspace = true tokio-stream = { version = "0.1.19", features = ["sync"] } tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs", "process", "signal", "time", "macros"] } -#console-subscriber = "0" -#tracing = "0.1" -#tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } tokio-util = { version = "0.7.19", features = ["io", "io-util"] } tempfile = "3.27.0" base64.workspace = true @@ -99,7 +97,7 @@ windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Stora # tikv-jemallocator = { version = "*", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } [build-dependencies] -vergen = { version = "9.1.0", features = ["build"] } +vergen = { version = "10.0.3", features = ["build"] } [dev-dependencies] tuliprox-processing = { workspace = true, features = ["test-support"] } @@ -121,5 +119,5 @@ proxy-auth-regression = [] [[bin]] -name = "flags_builder" +name = "flags-builder" path = "src/tools/flags_builder.rs" diff --git a/backend/app/build.rs b/backend/app/build.rs index 6775589f3..2ff26daa6 100644 --- a/backend/app/build.rs +++ b/backend/app/build.rs @@ -1,8 +1,8 @@ use std::error::Error; -use vergen::{BuildBuilder, Emitter}; +use vergen::{Build, Emitter}; fn main() -> Result<(), Box> { - let build = BuildBuilder::all_build()?; + let build = Build::all_build(); Emitter::default().add_instructions(&build)?.emit()?; Ok(()) } diff --git a/backend/app/src/api/api_utils/mod.rs b/backend/app/src/api/api_utils/mod.rs index 5ba652a99..94fb7efd6 100644 --- a/backend/app/src/api/api_utils/mod.rs +++ b/backend/app/src/api/api_utils/mod.rs @@ -7,12 +7,12 @@ use crate::{ endpoints::xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext}, model::{ create_active_client_stream, create_channel_unavailable_stream, create_custom_video_stream_response, - create_provider_connections_exhausted_stream, create_provider_stream, - get_custom_stream_response_error_status, get_stream_response_with_headers, is_custom_video_stream_enabled, - tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, - ProviderAllocation, ProviderConfig, ProviderHandle, ProviderStreamCustomReason, - ProviderStreamFactoryOptions, ProviderStreamInfo, ProviderStreamState, SharedStreamCtx, - SharedStreamManager, StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, + create_provider_connections_exhausted_stream, get_custom_stream_response_error_status, + get_stream_response_with_headers, is_custom_video_stream_enabled, open_provider_stream_with_lifecycle, + tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, PlaybackLeaseRef, + ProviderAllocation, ProviderConfig, ProviderStreamCustomReason, ProviderStreamFactoryOptions, + ProviderStreamInfo, ProviderStreamOpenLifecycle, ProviderStreamState, SharedStreamCtx, SharedStreamManager, + StreamAdmissionError, StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, }, }, @@ -25,7 +25,9 @@ use crate::{ }, MediaServerError, MediaServerErrorKind, MediaServerHttpClient, MediaServerImageRef, }, - model::{AppConfig, ConfigInput, ConfigInputFlags, ConfigTarget, InputUserInfo, ProxyUserCredentials}, + model::{ + AppConfig, ConfigInput, ConfigInputFlags, ConfigTarget, InputUserInfo, PlaybackKind, ProxyUserCredentials, + }, processing::{ parser::hls::{rewrite_hls, RewriteHlsProps}, processor::re_resolve_stalker_url, @@ -82,12 +84,12 @@ use tuliprox_hls::api::MAX_HLS_MANIFEST_BYTES; use url::Url; pub(crate) fn resolve_request_url_for_logging<'a>(input: &ConfigInput, stream_url: &'a str) -> Cow<'a, str> { + if !is_sanitize_sensitive_info_enabled() { + return Cow::Borrowed(stream_url); + } if is_media_server_playback_url(input, stream_url) { return Cow::Borrowed("media-server://"); } - if is_sanitize_sensitive_info_enabled() { - return Cow::Borrowed(stream_url); - } let provider = input.get_resolve_provider(stream_url); if let Ok(url) = Url::parse(stream_url) { @@ -183,6 +185,21 @@ pub(crate) fn admission_failure_response( create_custom_video_stream_response(&app_state.provider_stream_ctx(), &fingerprint.addr, video_type).into_response() } +/// Produces a defined non-success response for a rejected playback request. Unlike the +/// upstream provider response, this carries no Content-Length/Content-Range so downstream +/// players receive an unambiguous error instead of an empty success body. +fn stream_admission_rejected_response(error: StreamAdmissionError, username: &str) -> axum::response::Response { + error!("Stream admission rejected for user {username}: {error:?}"); + axum::response::Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .header("x-tuliprox-rejection", "admission_rejected") + .body(axum::body::Body::empty()) + .unwrap_or_else(|_| StatusCode::SERVICE_UNAVAILABLE.into_response()) +} + +/// Produces a quiet termination response when a recently evicted stream retries. +pub fn reentry_suppressed_response() -> axum::response::Response { StatusCode::NO_CONTENT.into_response() } + #[macro_export] macro_rules! try_option_bad_request { ($option:expr, $msg_is_error:expr, $msg:expr) => { @@ -349,10 +366,19 @@ pub(crate) use tuliprox_core::utils::response_compression::{ mark_response_as_uncompressed, should_compress_response_extensions, }; -#[derive(Clone, Copy, Debug, Default)] +#[derive(Default)] struct StreamMeteringConfig { meter_uid: u32, meter_stream: bool, + pending_shared_registration: Option, +} + +impl StreamMeteringConfig { + fn commit_shared_registration(&mut self) { + if let Some(registration) = self.pending_shared_registration.take() { + registration.commit(); + } + } } #[allow(clippy::missing_panics_doc)] @@ -439,6 +465,7 @@ struct StreamingAcquireOptions<'a> { user_priority: i8, connection_kind: crate::api::model::ConnectionKind, session_owner: Option<&'a str>, + playback_kind: PlaybackKind, accept_requested_stream_url: bool, } @@ -493,7 +520,7 @@ async fn activate_session_before_stream_open( // Classify based on current session state, not the pre-computed value. // If caller passes FollowUp, verify the session is still counted under the guard. // A stale FollowUp would bypass admission β€” reclassify to catch this. - let effective_request_class = if let Some(request_class) = request_class { + let (effective_request_class, loaded_session) = if let Some(request_class) = request_class { if matches!(request_class, PlaybackRequestClass::FollowUp | PlaybackRequestClass::Activate) { // Re-read session under the guard to ensure the counted lease is still held or acquired. // If it is no longer counted, classify it from the current lifecycle so @@ -501,21 +528,23 @@ async fn activate_session_before_stream_open( // If it became counted, classify it so stale Activate requests don't double count. let current_session = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - classify_playback_request(PlaybackRequestFacts { + let classified = classify_playback_request(PlaybackRequestFacts { existing_session: current_session.as_ref(), prepare_only: false, terminate: false, - }) + }); + (classified, Some(current_session)) } else { - request_class + (request_class, None) } } else { let existing_session = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - classify_playback_request(PlaybackRequestFacts { + let classified = classify_playback_request(PlaybackRequestFacts { existing_session: existing_session.as_ref(), prepare_only: false, terminate: false, - }) + }); + (classified, Some(existing_session)) }; let limits_enabled = app_state.app_config.config.load().user_access_control && (user.max_connections > 0 || user.soft_connections > 0); @@ -529,7 +558,10 @@ async fn activate_session_before_stream_open( if connection_permission == UserConnectionPermission::GracePeriod { // Materialize grace lifecycle under the guard so the session state is consistent. // Determine which grace mode applies by checking the current session state. - let current_session = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; + let current_session = match loaded_session { + Some(session) => session, + None => app_state.active_users.get_and_update_user_session(&user.username, session_token).await, + }; let (_, resolved_grace) = match current_session.as_ref().map(|s| &s.lifecycle) { Some(crate::api::model::PlaybackLifecycle::PendingProvider { .. }) => { // Session already in PendingProvider β€” refresh deadline. @@ -599,10 +631,10 @@ async fn activate_session_before_stream_open( } }; return PlaybackActivationResult { - admission: crate::api::model::ConnectionAdmission { - permission: connection_permission, - kind: Some(connection_kind), - }, + admission: crate::api::model::ConnectionAdmission::from_permission( + connection_permission, + Some(connection_kind), + ), grace_mode: resolved_grace, grace_context: None, placeholder_transition_version: None, @@ -614,10 +646,10 @@ async fn activate_session_before_stream_open( || effective_request_class == PlaybackRequestClass::Prepare { return PlaybackActivationResult { - admission: crate::api::model::ConnectionAdmission { - permission: connection_permission, - kind: Some(connection_kind), - }, + admission: crate::api::model::ConnectionAdmission::from_permission( + connection_permission, + Some(connection_kind), + ), grace_mode: None, grace_context: None, placeholder_transition_version: None, @@ -664,7 +696,7 @@ async fn activate_session_before_stream_open( let grace_mode = result.grace_mode; let grace_context = result.grace_context; - if admission.permission == UserConnectionPermission::GracePeriod { + if admission.permission() == UserConnectionPermission::GracePeriod { if matches!(grace_mode, Some(crate::api::model::GraceMode::Hold)) { // Hold: session waits for provider slot. Does not count until provider is acquired. let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); @@ -1141,24 +1173,26 @@ async fn acquire_stream_provider_handle( app_state: &Arc, input: &ConfigInput, fingerprint: &Fingerprint, - options: StreamingAcquireOptions<'_>, -) -> Option { + options: &StreamingAcquireOptions<'_>, +) -> Option { + let lease = options.session_owner.map(|owner| PlaybackLeaseRef::new(owner, options.playback_kind)); + let managed = |handle| tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle); match options.force_provider { Some(provider) => { // First try to stay on the exact pinned provider account without over-allocating. if let Some(handle) = app_state .active_provider - .acquire_exact_connection_with_grace_for_session( + .acquire_exact_connection_with_lease_for_session_await( provider, &fingerprint.addr, options.allow_provider_grace, options.user_priority, options.connection_kind, - options.session_owner, + lease, ) .await { - Some(handle) + Some(managed(handle)) } else if options.allow_forced_provider_fallback { debug_if_enabled!( "Pinned provider {} unavailable for {}; falling back to lineup allocation", @@ -1167,15 +1201,16 @@ async fn acquire_stream_provider_handle( ); app_state .active_provider - .acquire_connection_with_grace_for_session( + .acquire_connection_with_lease_for_session_await( &input.name, &fingerprint.addr, options.allow_provider_grace, options.user_priority, options.connection_kind, - options.session_owner, + lease, ) .await + .map(managed) } else { debug_if_enabled!( "Pinned provider {} unavailable for {}; strict provider affinity prevents fallback", @@ -1185,19 +1220,18 @@ async fn acquire_stream_provider_handle( None } } - None => { - app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &fingerprint.addr, - options.allow_provider_grace, - options.user_priority, - options.connection_kind, - options.session_owner, - ) - .await - } + None => app_state + .active_provider + .acquire_connection_with_lease_for_session_await( + &input.name, + &fingerprint.addr, + options.allow_provider_grace, + options.user_priority, + options.connection_kind, + lease, + ) + .await + .map(managed), } } @@ -1208,13 +1242,9 @@ pub(crate) fn resolve_redirect_location<'a>( input.map_or(Ok(Cow::Borrowed(stream_url)), |input| input.resolve_url(stream_url)) } -async fn get_redirect_alternative_url( - app_state: &Arc, - redirect_url: &Arc, - input: &ConfigInput, -) -> Arc { +fn get_redirect_alternative_url(app_state: &Arc, redirect_url: &Arc, input: &ConfigInput) -> Arc { if let Some((base_url, username, password)) = input.get_matched_config_by_url(redirect_url) { - if let Some(provider_cfg) = app_state.active_provider.get_next_provider(&input.name).await { + if let Some(provider_cfg) = app_state.active_provider.get_next_provider(&input.name) { let mut new_url = redirect_url.replacen(base_url, provider_cfg.url.as_str(), 1); if let (Some(old_username), Some(old_password)) = (username, password) { if let (Some(new_username), Some(new_password)) = @@ -1258,12 +1288,13 @@ async fn resolve_streaming_strategy( ) -> StreamingStrategy { // allocate a provider connection let accept_requested_stream_url = options.accept_requested_stream_url || input.input_type.is_stalker(); - let mut provider_connection_handle = acquire_stream_provider_handle(app_state, input, fingerprint, options).await; + let mut provider_connection_handle = acquire_stream_provider_handle(app_state, input, fingerprint, &options).await; // panel_api provisioning/loading is handled later in the stream creation flow let mut release_failed_mapping = false; - let stream_response_params = if let Some(allocation) = provider_connection_handle.as_ref().map(|ph| &ph.allocation) + let stream_response_params = if let Some(allocation) = + provider_connection_handle.as_ref().and_then(|managed| managed.handle()).map(|ph| &ph.allocation) { match allocation { ProviderAllocation::Exhausted => { @@ -1319,12 +1350,9 @@ async fn resolve_streaming_strategy( }; if release_failed_mapping { - if let Some(handle) = provider_connection_handle.take() { - let connection_manager = Arc::clone(&app_state.connection_manager); - tokio::spawn(async move { - connection_manager.release_provider_handle(Some(handle)).await; - }); - } + // The managed owner releases the provider slot synchronously on drop; no + // task-per-drop and no lossy cleanup message. + drop(provider_connection_handle.take()); } StreamingStrategy { @@ -1490,6 +1518,7 @@ async fn create_stream_response_details( user_priority, connection_kind, session_owner, + playback_kind: PlaybackKind::classify(item_type, extract_extension_from_url(stream_url)), accept_requested_stream_url, }, ) @@ -1514,16 +1543,19 @@ async fn create_stream_response_details( let provider_grace_active = matches!(streaming_strategy.provider_stream_state, ProviderStreamState::GracePeriod(_, _)); - let guard_provider_name = - streaming_strategy.provider_handle.as_ref().and_then(|guard| guard.allocation.get_provider_name()); + let guard_provider_name = streaming_strategy + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.allocation.get_provider_name()); if matches!( streaming_strategy.provider_stream_state, ProviderStreamState::Custom { reason: ProviderStreamCustomReason::ProviderExhausted, .. } ) && can_provision_on_exhausted(app_state, input) { - if let Some(handle) = streaming_strategy.provider_handle.take() { - app_state.connection_manager.release_provider_handle(Some(handle)).await; + if let Some(managed) = streaming_strategy.provider_handle.take() { + drop(managed); } debug_if_enabled!( "panel_api: provider connections exhausted; sending provisioning stream for input {}", @@ -1543,12 +1575,13 @@ async fn create_stream_response_details( match streaming_strategy.provider_stream_state { // custom stream means we display our own stream like connection exhausted, channel-unavailable... - ProviderStreamState::Custom { response: provider_stream, .. } => { + ProviderStreamState::Custom { response: provider_stream, reason } => { let (stream, stream_info) = provider_stream; // When allocation is exhausted or no connection was acquired, guard_provider_name is None. // Use input.name as fallback so the provider field is never empty. let provider_name = guard_provider_name.clone().unwrap_or_else(|| input.name.clone()); Ok(StreamDetails { + shared_subscriber_id: None, stream, stream_info, provider_name: Some(provider_name), @@ -1560,9 +1593,10 @@ async fn create_stream_response_details( provider_grace_active: false, disable_provider_grace: false, reconnect_flag: None, - provider_handle: streaming_strategy.provider_handle.clone(), + provider_handle: streaming_strategy.provider_handle.take(), content_representation, grace_resolution_context, + custom_reason: Some(reason), }) } ProviderStreamState::Available(_provider_name, request_url) @@ -1580,7 +1614,7 @@ async fn create_stream_response_details( is_reopen, ) { if let Some(provider_name) = guard_provider_name.as_ref() { - app_state.active_provider.is_over_limit(provider_name).await + app_state.active_provider.is_over_limit(provider_name) } else { false } @@ -1640,10 +1674,15 @@ async fn create_stream_response_details( } let reconnect_flag = provider_stream_factory_options.get_reconnect_flag_clone(); - let provider_stream = match create_provider_stream( + let lifecycle = streaming_strategy + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); + let provider_stream = match open_provider_stream_with_lifecycle( &app_state.provider_stream_ctx(), &app_state.http_client.load(), provider_stream_factory_options, + lifecycle, ) .await { @@ -1703,11 +1742,19 @@ async fn create_stream_response_details( } options.set_provider(input.get_resolve_provider(url.as_ref())); options.require_public_destination(); + if let Some(m) = streaming_strategy.provider_handle.as_mut() { + let _ = m.renew_opening_tokens(); + } let retry_reconnect_flag = options.get_reconnect_flag_clone(); - let retried = create_provider_stream( + let retry_lifecycle = streaming_strategy + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); + let retried = open_provider_stream_with_lifecycle( &app_state.provider_stream_ctx(), &app_state.http_client.load(), options, + retry_lifecycle, ) .await; if let Some(response) = retried { @@ -1748,9 +1795,12 @@ async fn create_stream_response_details( // An intentional deferred open must retain its grace allocation until body polling // resumes the provider request. Other failed opens release their allocation here. - let provider_handle = if stream.is_none() && !defer_provider_stream_until_grace_check { - let provider_handle = streaming_strategy.provider_handle.take(); - app_state.connection_manager.release_provider_handle(provider_handle).await; + let failed_open = stream.is_none() + || matches!(stream_info.as_ref(), Some((_, _, _, Some(custom))) if *custom != CustomVideoStreamType::Provisioning); + let provider_handle = if failed_open && !defer_provider_stream_until_grace_check { + // The managed owner releases the provider slot synchronously on drop; + // a failed open must not rely on a lossy cleanup message. + drop(streaming_strategy.provider_handle.take()); error!("Can't open stream {}", sanitize_sensitive_info(&request_url)); None } else { @@ -1758,6 +1808,7 @@ async fn create_stream_response_details( }; Ok(StreamDetails { + shared_subscriber_id: None, stream, stream_info, provider_name: guard_provider_name.clone(), @@ -1772,6 +1823,7 @@ async fn create_stream_response_details( provider_handle, content_representation, grace_resolution_context, + custom_reason: None, }) } } @@ -1839,7 +1891,7 @@ where }; let redirect_url = if is_dash_request { replace_url_extension(&redirect_url, DASH_EXT).into() } else { redirect_url }; - let redirect_url = get_redirect_alternative_url(app_state, &redirect_url, params.input).await; + let redirect_url = get_redirect_alternative_url(app_state, &redirect_url, params.input); let redirect_url = match resolve_redirect_location(Some(params.input), &redirect_url) { Ok(url) => url, Err(err) => { @@ -1855,7 +1907,7 @@ where return Some(StatusCode::BAD_REQUEST.into_response()); }; - if redirect_request { + if redirect_request || is_dash_request { let target_name = params.target.name.as_str(); let virtual_id = params.item.get_virtual_id(); let stream_url = match get_xtream_player_api_stream_url( @@ -1871,7 +1923,7 @@ where ); return Some(StatusCode::BAD_REQUEST.into_response()); } - Some(url) => match app_state.active_provider.get_next_provider(¶ms.input.name).await { + Some(url) => match app_state.active_provider.get_next_provider(¶ms.input.name) { Some(provider_cfg) => match get_stream_alternative_url(&url, params.input, &provider_cfg) { Some(stream_url) => stream_url, None => return Some(StatusCode::BAD_REQUEST.into_response()), @@ -2068,7 +2120,7 @@ pub async fn force_provider_stream_response( sanitize_sensitive_info(&user_session.token), sanitize_sensitive_info(&fingerprint.addr.to_string()) ); - cleanup_forced_reopen_addrs(app_state, item_type, &cleanup_addrs).await; + cleanup_forced_reopen_addrs(app_state, &user_session.token, &cleanup_addrs).await; } // Provider-affine playback must stay on the same provider account across seeks/range reconnects. @@ -2125,25 +2177,26 @@ pub async fn force_provider_stream_response( let metering = prepare_stream_metering( app_state, user_session.stream_url.as_ref(), - share_stream, + false, stream_details.stream.is_some(), stream_details.has_deferred_provider_open(), - ) - .await; + ); let provider_response = stream_details.stream_info.as_ref().map(|(h, sc, url, cvt)| (h.clone(), *sc, url.clone(), *cvt)); if ctx.session_reservation_ttl_secs > 0 { if let Some(provider_name) = stream_details.provider_name.as_ref() { - app_state - .active_provider - .refresh_provider_reservation(provider_name, &user_session.token, ctx.session_reservation_ttl_secs) - .await; + app_state.active_provider.refresh_adaptive_playback_lease( + provider_name, + &user_session.token, + PlaybackKind::classify(item_type, extract_extension_from_url(user_session.stream_url.as_ref())), + ctx.session_reservation_ttl_secs, + ); } } app_state.active_users.update_session_addr(&ctx.user.username, &user_session.token, &fingerprint.addr).await; stream_channel.shared = share_stream; let socket_bound = user_session.socket_bound; - let stream = create_active_client_stream(crate::api::model::ActiveClientStreamParams { + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { stream_details, app_state, user: ctx.user, @@ -2157,7 +2210,17 @@ pub async fn force_provider_stream_response( meter_uid: metering.meter_uid, meter_stream: metering.meter_stream, }) - .await; + .await + { + Ok(stream) => stream, + Err(error) => { + app_state + .active_users + .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) + .await; + return stream_admission_rejected_response(error, &ctx.user.username); + } + }; let (status_code, header_map) = get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); let mut response = axum::response::Response::builder().status(status_code); @@ -2177,7 +2240,7 @@ pub async fn force_provider_stream_response( return response; } - app_state.connection_manager.release_provider_handle(stream_details.provider_handle).await; + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle); app_state .active_users .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) @@ -2249,8 +2312,8 @@ pub(crate) async fn stream_response( ) .await; let grace_mode = activation.grace_mode.or(grace_mode); - connection_permission = activation.admission.permission; - connection_kind = activation.admission.kind.unwrap_or(connection_kind); + connection_permission = activation.admission.permission(); + connection_kind = activation.admission.kind().unwrap_or(connection_kind); let allow_shared_reuse = connection_permission != UserConnectionPermission::Exhausted || allow_exhausted_shared_reconnect; @@ -2270,6 +2333,7 @@ pub(crate) async fn stream_response( stream_channel.clone(), session_token, req_headers, + activation.placeholder_transition_version, ) .await { @@ -2291,6 +2355,7 @@ pub(crate) async fn stream_response( stream_channel.clone(), session_token, req_headers, + activation.placeholder_transition_version, ) .await { @@ -2311,6 +2376,9 @@ pub(crate) async fn stream_response( activation.placeholder_transition_version.is_some(), ) .await; + if activation.admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } record_connect_failed_attempt(ConnectFailedAttempt { app_state, fingerprint, @@ -2440,15 +2508,6 @@ pub(crate) async fn stream_response( debug_if_enabled!("panel_api provisioning response to client: status={} headers={:?}", status, headers); } - let metering = prepare_stream_metering( - app_state, - stream_url, - share_stream, - stream_details.stream.is_some(), - stream_details.has_deferred_provider_open(), - ) - .await; - // Captured before `stream_details` is moved into `create_active_client_stream`. // The pinning rule is centralized in `should_pin_provider_for_session` so it stays // testable in isolation and in sync with the call site below. @@ -2469,7 +2528,56 @@ pub(crate) async fn stream_response( is_stream_shared = false; } } + let shared_subscriber_id = + tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); + let mut pending_shared_cleanup = if is_stream_shared { + match SharedStreamManager::reserve_subscriber_cleanup( + &app_state.connection_manager, + shared_subscriber_id, + fingerprint.addr, + ) + .await + { + Ok(cleanup) => Some(cleanup), + Err(reason) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + return stream_admission_rejected_response(reason.into(), &user.username); + } + } + } else { + None + }; + let mut metering = prepare_stream_metering( + app_state, + stream_url, + is_stream_shared, + stream_details.stream.is_some(), + stream_details.has_deferred_provider_open(), + ); + if let (Some(cleanup), Some(request_id)) = ( + pending_shared_cleanup.as_mut(), + stream_details + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.playback_request_id), + ) { + cleanup.set_provider_request_identity(session_token, request_id); + } let provider_handle = if is_stream_shared && !stream_details.has_deferred_provider_open() { + // Transfer the ManagedProviderHandle to the shared-stream manager. + // Ownership stays with the managed type across all awaits; the shared-stream + // manager disarms it only once it holds the write lock and commits to a new + // shared origin. If the future is cancelled before that point the Drop guard + // releases the slot. stream_details.provider_handle.take() } else { None @@ -2483,7 +2591,11 @@ pub(crate) async fn stream_response( stream_channel.shared_joined_existing = None; stream_channel.shared_stream_id = None; } - let stream = create_active_client_stream(crate::api::model::ActiveClientStreamParams { + if is_stream_shared { + stream_details.shared_subscriber_id = + pending_shared_cleanup.as_ref().map(tuliprox_session::PendingSharedSubscriberCleanup::capability); + } + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { stream_details, app_state, user, @@ -2497,32 +2609,54 @@ pub(crate) async fn stream_response( meter_uid: metering.meter_uid, meter_stream: metering.meter_stream, }) - .await; + .await + { + Ok(stream) => stream, + Err(error) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + return stream_admission_rejected_response(error, &user.username); + } + }; let stream_resp = if is_stream_shared { debug_if_enabled!( "Streaming shared stream request from {}", sanitize_sensitive_info(log_actual_request_url.as_ref()) ); + let Some(pending_shared_cleanup) = pending_shared_cleanup else { + return stream_admission_rejected_response(StreamAdmissionError::RegistrationRejected, &user.username); + }; // Shared Stream response let shared_headers = provider_response.as_ref().map_or_else(Vec::new, |(h, _, _, _)| h.clone()); - if let Some((broadcast_stream, _shared_provider)) = SharedStreamManager::register_shared_stream( - SharedStreamCtx { - app_config: &app_state.app_config, - shared_stream_manager: &app_state.shared_stream_manager, - active_provider: &app_state.active_provider, - connection_manager: &app_state.connection_manager, - }, - stream_url, - stream, - &fingerprint.addr, - shared_headers, - stream_options.buffer_size, - provider_handle, - connection_priority_for_kind(user, connection_kind), - connection_kind, - ) - .await + if let Some((broadcast_stream, _shared_provider, _cleanup_capability)) = + SharedStreamManager::register_shared_stream( + SharedStreamCtx { + app_config: &app_state.app_config, + shared_stream_manager: &app_state.shared_stream_manager, + active_provider: &app_state.active_provider, + connection_manager: &app_state.connection_manager, + }, + stream_url, + stream, + &fingerprint.addr, + shared_subscriber_id, + shared_headers, + stream_options.buffer_size, + provider_handle, + pending_shared_cleanup, + connection_priority_for_kind(user, connection_kind), + connection_kind, + ) + .await { + metering.commit_shared_registration(); let (status_code, header_map) = get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); let mut response = axum::response::Response::builder().status(status_code); @@ -2618,10 +2752,12 @@ pub(crate) async fn stream_response( if should_pin_provider { let reservation_ttl_secs = get_session_reservation_ttl_secs(app_state, item_type); if reservation_ttl_secs > 0 { - app_state - .active_provider - .refresh_provider_reservation(&provider, session_token, reservation_ttl_secs) - .await; + app_state.active_provider.refresh_adaptive_playback_lease( + &provider, + session_token, + PlaybackKind::classify(item_type, playback_extension), + reservation_ttl_secs, + ); } } } @@ -2635,7 +2771,7 @@ pub(crate) async fn stream_response( return stream_resp.into_response(); } - app_state.connection_manager.release_provider_handle(stream_details.provider_handle).await; + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle); app_state .active_users .release_unbound_session_reservation( @@ -2645,6 +2781,24 @@ pub(crate) async fn stream_response( activation.placeholder_transition_version.is_some(), ) .await; + if stream_details.custom_reason == Some(ProviderStreamCustomReason::ProviderExhausted) { + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel, + provider_name: stream_details.provider_name.unwrap_or_else(|| input.name.clone()), + req_headers, + reason: ConnectFailureReason::ProviderConnectionsExhausted, + failure_stage: FailureStage::Admission, + }); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::ProviderConnectionsExhausted, + ) + .into_response(); + } no_custom_video_fallback_status(&app_state.app_config).into_response() } @@ -2749,11 +2903,13 @@ async fn detected_catchup_hls_response(params: DetectedCatchupHlsResponseParams< ) .await; } - app_state - .active_provider - .refresh_provider_reservation(&provider, &created_session_token, get_catchup_session_ttl_secs(app_state)) - .await; - app_state.connection_manager.release_provider_handle(stream_details.provider_handle.take()).await; + app_state.active_provider.refresh_adaptive_playback_lease( + &provider, + &created_session_token, + PlaybackKind::Catchup, + get_catchup_session_ttl_secs(app_state), + ); + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle.take()); app_state .active_users .release_unbound_session_reservation(&user.username, &created_session_token, None, false) @@ -2779,9 +2935,9 @@ async fn cleanup_failed_detected_catchup_hls( username: &str, session_token: &str, ) { - app_state.connection_manager.release_provider_handle(stream_details.provider_handle.take()).await; + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle.take()); app_state.active_users.terminate_session(username, session_token).await; - app_state.active_provider.clear_provider_reservation(session_token).await; + app_state.active_provider.clear_provider_reservation(session_token); } async fn probe_catchup_payload(stream: BoxedProviderStream, deadline: Duration) -> Result { @@ -2842,7 +2998,7 @@ fn is_stream_metrics_enabled(app_state: &Arc) -> bool { .is_some_and(|stream| stream.metrics_enabled) } -async fn prepare_stream_metering( +fn prepare_stream_metering( app_state: &Arc, stream_url: &str, share_stream: bool, @@ -2854,14 +3010,17 @@ async fn prepare_stream_metering( } if share_stream { - let meter_uid = app_state + let (meter_uid, pending_registration) = app_state .shared_stream_manager - .get_or_register_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()) - .await; - return StreamMeteringConfig { meter_uid, meter_stream: has_stream || has_deferred_provider_open }; + .reserve_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()); + return StreamMeteringConfig { + meter_uid, + meter_stream: has_stream || has_deferred_provider_open, + pending_shared_registration: pending_registration, + }; } else if has_stream || has_deferred_provider_open { let meter_uid = app_state.connection_manager.next_stream_uid(); - return StreamMeteringConfig { meter_uid, meter_stream: true }; + return StreamMeteringConfig { meter_uid, meter_stream: true, pending_shared_registration: None }; } StreamMeteringConfig::default() @@ -2889,18 +3048,8 @@ pub(crate) fn get_hls_session_ttl_secs(app_state: &Arc) -> u64 { get_stream_config_u64(app_state, |stream| stream.hls_session_ttl_secs, default_hls_session_ttl_secs()) } -async fn cleanup_forced_reopen_addrs( - app_state: &Arc, - item_type: PlaylistItemType, - cleanup_addrs: &[SocketAddr], -) { - let close_client_socket = !(item_type.is_live_adaptive() || item_type == PlaylistItemType::Catchup); - for addr in cleanup_addrs { - app_state.connection_manager.release_provider_connection(addr).await; - if close_client_socket { - let _ = app_state.connection_manager.close_connection_signal(addr); - } - } +async fn cleanup_forced_reopen_addrs(app_state: &Arc, session_owner: &str, cleanup_addrs: &[SocketAddr]) { + app_state.active_provider.release_playback_connections_await(session_owner, cleanup_addrs).await; } pub(crate) fn get_catchup_session_ttl_secs(app_state: &Arc) -> u64 { @@ -2938,7 +3087,7 @@ pub(crate) fn should_pin_provider_for_session( ) } -#[allow(clippy::too_many_arguments)] +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn try_shared_stream_response_if_any( app_state: &Arc, stream_url: &str, @@ -2949,8 +3098,11 @@ async fn try_shared_stream_response_if_any( mut stream_channel: StreamChannel, session_token: &str, req_headers: &HeaderMap, + placeholder_transition_version: Option, ) -> Option { - if let Some((stream, provider)) = SharedStreamManager::subscribe_shared_stream( + let subscriber_id = + tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); + let shared_subscription = SharedStreamManager::subscribe_shared_stream( SharedStreamCtx { app_config: &app_state.app_config, shared_stream_manager: &app_state.shared_stream_manager, @@ -2959,71 +3111,116 @@ async fn try_shared_stream_response_if_any( }, stream_url, &fingerprint.addr, + subscriber_id, connection_priority_for_kind(user, connection_kind), connection_kind, ) - .await - { - debug_if_enabled!("Using shared stream {}", sanitize_sensitive_info(stream_url)); - if let Some(headers) = app_state.shared_stream_manager.get_shared_state_headers(stream_url).await { - let (status_code, header_map) = get_stream_response_with_headers(Some((headers.clone(), StatusCode::OK))); - let mut grace_period_options = app_state.get_grace_options(); - if connect_permission != UserConnectionPermission::GracePeriod { - grace_period_options.period_millis = 0; - } - let mut stream_details = StreamDetails::from_stream(stream, grace_period_options); - - stream_details.provider_name = provider; - let socket_bound = - is_socket_bound_playback_session(stream_channel.item_type, extract_extension_from_url(stream_url)); - if let Some(provider_name) = stream_details.provider_name.as_deref() { - let _ = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id: stream_channel.virtual_id, - provider: provider_name, - stream_url, - addr: &fingerprint.addr, - connection_permission: connect_permission, - connection_kind: Some(connection_kind), - socket_bound, - }) - .await; - } - stream_channel.shared = true; - stream_channel.shared_joined_existing = Some(true); - let meter_uid = app_state - .shared_stream_manager - .get_or_register_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()) + .await; + let (stream, provider, cleanup_capability) = match shared_subscription { + Ok(Some(subscription)) => subscription, + Ok(None) => return None, + Err(reason) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + placeholder_transition_version, + placeholder_transition_version.is_some(), + ) .await; - stream_channel.shared_stream_id = Some(u64::from(meter_uid)); - let metering = StreamMeteringConfig { meter_uid, meter_stream: false }; - let stream = create_active_client_stream(crate::api::model::ActiveClientStreamParams { - stream_details, - app_state, - user, - connection_permission: connect_permission, - connection_kind, - fingerprint, - stream_channel, - socket_bound, - session_token: Some(session_token), - req_headers, - meter_uid: metering.meter_uid, - meter_stream: metering.meter_stream, - }) - .await - .boxed(); - let mut response = axum::response::Response::builder().status(status_code); - for (key, value) in &header_map { - response = response.header(key, value); - } - let mut response = response.body(axum::body::Body::from_stream(stream)).ok()?; - mark_response_as_uncompressed(&mut response); - return Some(response); + return Some(stream_admission_rejected_response(reason.into(), &user.username)); } + }; + debug_if_enabled!("Using shared stream {}", sanitize_sensitive_info(stream_url)); + if let Some(headers) = app_state.shared_stream_manager.get_shared_state_headers(stream_url).await { + let (status_code, header_map) = get_stream_response_with_headers(Some((headers.clone(), StatusCode::OK))); + let mut grace_period_options = app_state.get_grace_options(); + if connect_permission != UserConnectionPermission::GracePeriod { + grace_period_options.period_millis = 0; + } + let mut stream_details = StreamDetails::from_stream(stream, grace_period_options); + stream_details.shared_subscriber_id = Some(cleanup_capability); + + stream_details.provider_name = provider; + let socket_bound = + is_socket_bound_playback_session(stream_channel.item_type, extract_extension_from_url(stream_url)); + // A shared origin may have no provider allocation (e.g. a local/unknown owner), in + // which case `provider` is `None`. The subscriber must still be tracked as a user + // session, so fall back to a stable placeholder name rather than skipping session + // creation; `is_over_limit` on an unknown provider is a no-op. + let provider_name = stream_details.provider_name.as_deref().unwrap_or("shared"); + let _ = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id: stream_channel.virtual_id, + provider: provider_name, + stream_url, + addr: &fingerprint.addr, + connection_permission: connect_permission, + connection_kind: Some(connection_kind), + socket_bound, + }) + .await; + stream_channel.shared = true; + stream_channel.shared_joined_existing = Some(true); + // Joining an existing origin reuses its meter identity. When metrics are disabled + // no meter exists to reserve; reserving one here would leave an ownerless entry that + // teardown cannot match and remove. + let (meter_uid, pending_registration) = if is_stream_metrics_enabled(app_state) { + app_state + .shared_stream_manager + .reserve_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()) + } else { + (0, None) + }; + stream_channel.shared_stream_id = Some(u64::from(meter_uid)); + // The pending registration is carried through admission so its drop guard rolls + // the reservation back if the client stream is never created. + let mut metering = + StreamMeteringConfig { meter_uid, meter_stream: false, pending_shared_registration: pending_registration }; + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { + stream_details, + app_state, + user, + connection_permission: connect_permission, + connection_kind, + fingerprint, + stream_channel, + socket_bound, + session_token: Some(session_token), + req_headers, + meter_uid: metering.meter_uid, + meter_stream: metering.meter_stream, + }) + .await + { + Ok(stream) => { + metering.commit_shared_registration(); + stream.boxed() + } + Err(error) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + placeholder_transition_version, + placeholder_transition_version.is_some(), + ) + .await; + return Some(stream_admission_rejected_response(error, &user.username)); + } + }; + let mut response = axum::response::Response::builder().status(status_code); + for (key, value) in &header_map { + response = response.header(key, value); + } + let mut response = response.body(axum::body::Body::from_stream(stream)).ok()?; + mark_response_as_uncompressed(&mut response); + return Some(response); } None } @@ -3222,8 +3419,8 @@ pub(crate) async fn local_stream_response( ) .await; grace_mode = activation.grace_mode; - connection_permission = activation.admission.permission; - connection_kind = activation.admission.kind.unwrap_or(connection_kind); + connection_permission = activation.admission.permission(); + connection_kind = activation.admission.kind().unwrap_or(connection_kind); if connection_permission == UserConnectionPermission::Exhausted { app_state @@ -3235,6 +3432,9 @@ pub(crate) async fn local_stream_response( activation.placeholder_transition_version.is_some(), ) .await; + if activation.admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } return create_custom_video_stream_response( &app_state.provider_stream_ctx(), &fingerprint.addr, @@ -3276,8 +3476,8 @@ pub(crate) async fn local_stream_response( }) .await; } - let metering = prepare_stream_metering(app_state, &pli.url, false, true, false).await; - let stream = create_active_client_stream(crate::api::model::ActiveClientStreamParams { + let metering = prepare_stream_metering(app_state, &pli.url, false, true, false); + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { stream_details: StreamDetails::from_stream(stream, grace_period_options), app_state, user, @@ -3291,7 +3491,11 @@ pub(crate) async fn local_stream_response( meter_uid: metering.meter_uid, meter_stream: metering.meter_stream, }) - .await; + .await + { + Ok(stream) => stream, + Err(error) => return stream_admission_rejected_response(error, &user.username), + }; let mut response = Response::new(axum::body::Body::from_stream(stream)); @@ -3832,16 +4036,30 @@ pub(crate) fn create_playback_session_fingerprint( extension: Option<&str>, ) -> String { // This scopes the session identity, not the session address-tracking policy. - // Adaptive playlist starts need a per-initial-socket token so two players behind - // the same IP/UA can watch the same HLS/DASH stream independently. The created - // UserSession itself can still be non-socket-bound. + // Adaptive and seekable playback use the stable client fingerprint so follow-up + // requests on new sockets reuse the same logical session. Plain live playback + // remains socket-bound so a separate connection creates a separate session. let session_bound = is_session_based_playback(item_type, extension); let socket_bound = !session_bound && is_socket_bound_playback_session(item_type, extension); create_session_fingerprint(fingerprint, username, virtual_id, socket_bound) } -pub fn create_catchup_session_key(fingerprint: &Fingerprint, username: &str, virtual_id: u32) -> String { - concat_string!("catchup|", &fingerprint.key, "|", username, "|", &virtual_id.to_string(), "|session") +pub fn create_catchup_session_key( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + archive_discriminator: &str, +) -> String { + concat_string!( + "catchup|", + &fingerprint.key, + "|", + username, + "|", + &virtual_id.to_string(), + "|", + archive_discriminator.trim_matches('/') + ) } pub fn create_m3u_catchup_session_key( diff --git a/backend/app/src/api/api_utils/tests.rs b/backend/app/src/api/api_utils/tests.rs index e66877c5c..ae0738b5b 100644 --- a/backend/app/src/api/api_utils/tests.rs +++ b/backend/app/src/api/api_utils/tests.rs @@ -14,7 +14,12 @@ use crate::{ utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; -use axum::http::{HeaderMap, Response, StatusCode}; +use axum::{ + extract::{ConnectInfo, State}, + http::{HeaderMap, Response, StatusCode}, + routing::get, + Router, +}; use bytes::Bytes; use futures::stream; use http_body_util::BodyExt; @@ -23,23 +28,21 @@ use shared::{ foundation::Filter, model::{ AdmissionStrategy, ClusterFlags, ConfigPaths, ConfigProviderDto, ConfigTargetOptions, GeoIpUnavailablePolicy, - InputFetchMethod, InputType, PlaylistItemType, ProcessingOrder, ProviderUrlSelectionPolicy, ProxyType, - StreamChannel, XtreamCluster, + InputFetchMethod, InputType, PlaylistItem, PlaylistItemHeader, PlaylistItemType, ProcessingOrder, + ProviderUrlSelectionPolicy, ProxyType, StreamChannel, TargetType, XtreamCluster, }, utils::Internable, }; use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; use tokio::{ - io::AsyncWriteExt, + io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, sync::{mpsc, RwLock}, }; use tuliprox_core::utils::response_compression::should_compress_response; use tuliprox_session::{ - admission::{ - evaluate_remaining_strategies_after_grace, get_effective_admission_strategies, RECENT_EVICTION_REENTRY_TTL_SECS, - }, - GraceResolutionContext, + admission::{evaluate_remaining_strategies_after_grace, get_effective_admission_strategies}, + AdmissionRejectionReason, GraceResolutionContext, }; #[test] @@ -58,6 +61,22 @@ fn initial_stalker_playback_resolves_only_empty_urls() { assert_eq!(stalker_stream_kind(XtreamCluster::Live, PlaylistItemType::Catchup), StalkerStreamKind::Archive); } +#[test] +fn stream_admission_rejection_is_503_without_upstream_headers() { + use crate::api::model::StreamAdmissionError; + + for error in [ + StreamAdmissionError::CleanupReceiverClosed, + StreamAdmissionError::CleanupAdmissionTimeout, + StreamAdmissionError::RegistrationRejected, + ] { + let response = stream_admission_rejected_response(error, "test-user"); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().get("content-length").is_none(), "must not leak upstream content-length"); + assert!(response.headers().get("content-range").is_none(), "must not leak upstream content-range"); + } +} + fn test_runtime_provider(url: &str, username: &str, password: &str) -> Arc { test_runtime_provider_with_type(url, username, password, InputType::Xtream) } @@ -171,7 +190,7 @@ fn test_runtime_provider_with_type( }; Arc::new(RuntimeProviderConfig::new( &input, - Arc::new(RwLock::new(ProviderConfigConnection::default())), + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), Arc::new(|_, _| {}), )) } @@ -180,7 +199,7 @@ fn test_runtime_provider_without_credentials(url: &str, input_type: InputType) - let input = ConfigInput { name: "provider".intern(), url: url.to_string(), input_type, ..ConfigInput::default() }; Arc::new(RuntimeProviderConfig::new( &input, - Arc::new(RwLock::new(ProviderConfigConnection::default())), + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), Arc::new(|_, _| {}), )) } @@ -593,7 +612,7 @@ async fn select_provider_stream_url_uses_persisted_alias_url_for_independent_str }; let alias = Arc::new(RuntimeProviderConfig::new( &alias_input, - Arc::new(RwLock::new(ProviderConfigConnection::default())), + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), Arc::new(|_, _| {}), )); @@ -852,6 +871,33 @@ fn media_server_playback_urls_are_proxy_only_redirect_guard_candidates() { ); } +#[test] +fn test_resolve_request_url_for_logging_respects_sanitization_setting() { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _guard = LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + struct SanitizeSettingGuard(bool); + impl Drop for SanitizeSettingGuard { + fn drop(&mut self) { shared::utils::set_sanitize_sensitive_info(self.0); } + } + + let prev = shared::utils::is_sanitize_sensitive_info_enabled(); + let _restore = SanitizeSettingGuard(prev); + + let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; + let media_ref = "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fsecret_key"; + let standard_url = "http://provider.example/live/stream.m3u8"; + + // 1. When sanitization is enabled: + shared::utils::set_sanitize_sensitive_info(true); + assert_eq!(resolve_request_url_for_logging(&m3u_input, media_ref).as_ref(), "media-server://"); + assert_eq!(resolve_request_url_for_logging(&m3u_input, standard_url).as_ref(), standard_url); + + // 2. When sanitization is disabled: + shared::utils::set_sanitize_sensitive_info(false); + assert_eq!(resolve_request_url_for_logging(&m3u_input, media_ref).as_ref(), media_ref); + assert_eq!(resolve_request_url_for_logging(&m3u_input, standard_url).as_ref(), standard_url); +} + #[test] fn test_streaming_response_extension_disables_compression() { let mut response = Response::new(()); @@ -884,6 +930,139 @@ async fn spawn_legacy_hls_test_origin( (origin_addr, origin_task) } +async fn spawn_range_aware_test_origin( + body: &'static [u8], + connections: usize, +) -> (SocketAddr, tokio::task::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let origin_addr = listener.local_addr().expect("test origin address"); + let origin_task = tokio::spawn(async move { + let mut requests = Vec::new(); + for _ in 0..connections { + let Ok((mut socket, _)) = listener.accept().await else { break }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.expect("test origin reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + let request_lower = String::from_utf8_lossy(&request).to_ascii_lowercase(); + requests.push(request_lower.clone()); + + let range = request_lower + .lines() + .find_map(|line| line.strip_prefix("range: bytes=")) + .and_then(|spec| spec.split_once('-')) + .and_then(|(start, end)| { + let start: usize = start.trim().parse().ok()?; + let end: usize = + if end.trim().is_empty() { body.len().saturating_sub(1) } else { end.trim().parse().ok()? }; + Some((start, end.min(body.len().saturating_sub(1)))) + }) + .unwrap_or((0, body.len().saturating_sub(1))); + + let slice = &body[range.0..=range.1]; + let head = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp2t\r\nContent-Range: bytes {}-{}/{}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + range.0, + range.1, + body.len(), + slice.len() + ); + socket.write_all(head.as_bytes()).await.expect("test origin writes response headers"); + socket.write_all(slice).await.expect("test origin writes response body"); + } + requests + }); + (origin_addr, origin_task) +} + +#[allow(dead_code)] +#[derive(Clone)] +enum FakeOriginMode { + EmptyBody, + BlockFirstChunk(Arc), + ErrorBeforeFirstChunk, + ErrorAfterFirstChunk(Vec), + FixedBytes(Vec), +} + +async fn spawn_controlled_fake_origin( + mode: FakeOriginMode, + max_requests: usize, +) -> (SocketAddr, tokio::task::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let origin_addr = listener.local_addr().expect("test origin address"); + let origin_task = tokio::spawn(async move { + let mut requests = Vec::new(); + for _ in 0..max_requests { + let Ok((mut socket, _)) = listener.accept().await else { break }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let Ok(read) = socket.read(&mut chunk).await else { break }; + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + if request.is_empty() { + continue; + } + let request_lower = String::from_utf8_lossy(&request).to_ascii_lowercase(); + requests.push(request_lower); + + match &mode { + FakeOriginMode::EmptyBody => { + let head = + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::BlockFirstChunk(notify) => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let mut probe = [0_u8; 1]; + tokio::select! { + _ = notify.notified() => {}, + _ = socket.read(&mut probe) => {}, + } + let _ = socket.shutdown().await; + } + FakeOriginMode::ErrorBeforeFirstChunk => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::ErrorAfterFirstChunk(chunk) => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let chunk_hdr = format!("{:x}\r\n", chunk.len()); + let _ = socket.write_all(chunk_hdr.as_bytes()).await; + let _ = socket.write_all(chunk).await; + let _ = socket.write_all(b"\r\n").await; + let _ = socket.flush().await; + let _ = socket.shutdown().await; + } + FakeOriginMode::FixedBytes(bytes) => { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(bytes).await; + let _ = socket.shutdown().await; + } + } + } + requests + }); + (origin_addr, origin_task) +} + async fn forced_legacy_hls_test_response( origin_addr: SocketAddr, request_headers: &HeaderMap, @@ -1007,6 +1186,2583 @@ async fn forced_hls_unencoded_partial_response_preserves_range_and_disables_comp assert!(request.contains("\r\nrange: bytes=2-\r\n")); } +#[tokio::test] +async fn forced_reopen_stays_on_pinned_provider_account() { + const ACCOUNT_A_BODY: &[u8] = b"account-a-marker"; + const ACCOUNT_B_BODY: &[u8] = b"account-b-marker"; + + let head_a = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + ACCOUNT_A_BODY.len() + ); + let head_b = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + ACCOUNT_B_BODY.len() + ); + let (origin_a, task_a) = spawn_legacy_hls_test_origin(head_a, ACCOUNT_A_BODY.to_vec()).await; + let (origin_b, task_b) = spawn_legacy_hls_test_origin(head_b, ACCOUNT_B_BODY.to_vec()).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_a}"), + username: Some("user-a".to_string()), + password: Some("pass-a".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: format!("http://{origin_b}"), + username: Some("user-b".to_string()), + password: Some("pass-b".to_string()), + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let app_config = Arc::new(create_test_provider_app_config()); + app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(app_config); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_400)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + let session = UserSession { + token: "stickiness-token".to_string(), + transition_version: 1, + virtual_id: 41, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_a}/live/1.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + let mut stream_channel = create_test_live_channel(&format!("http://{origin_a}/live/1.ts")); + stream_channel.provider_id = 1; + stream_channel.input_name = Arc::clone(&input.name); + stream_channel.item_type = PlaylistItemType::Catchup; + stream_channel.cluster = XtreamCluster::Live; + stream_channel.url = session.stream_url.clone(); + + let response = force_provider_stream_response( + &fingerprint, + &app_state, + &session, + stream_channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.expect("stream body").to_bytes(); + assert_eq!(body.as_ref(), ACCOUNT_A_BODY, "forced reopen must stay on the pinned provider account A"); + + let request_a = task_a.await.expect("origin A task completes").to_ascii_lowercase(); + assert!(!request_a.is_empty(), "origin A must have received the seek request"); + + let request_b = tokio::time::timeout(std::time::Duration::from_millis(250), task_b).await; + assert!(request_b.is_err(), "origin B must not receive a seek from an A-affine session"); +} + +#[tokio::test] +async fn overlapping_vod_range_requests_return_correct_account_bytes() { + const VOD_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(VOD_BODY, 2).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_500)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/movie/1.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/movie/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = format!("http://{origin_addr}/movie/1.ts").intern(); + channel + }; + + let build_request = |range: &'static str| { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static(range)); + let session = make_session(match range { + "bytes=0-9" => "vod-range-1", + _ => "vod-range-2", + }); + (headers, session) + }; + + let (first_headers, first_session) = build_request("bytes=0-9"); + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &first_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(first.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), Some("bytes 0-9/20")); + let first_body = first.into_body().collect().await.expect("first range body").to_bytes(); + assert_eq!(first_body.as_ref(), &VOD_BODY[0..10]); + + let (second_headers, second_session) = build_request("bytes=5-14"); + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &second_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + second.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), + Some("bytes 5-14/20") + ); + let second_body = second.into_body().collect().await.expect("second range body").to_bytes(); + assert_eq!(second_body.as_ref(), &VOD_BODY[5..15]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 2, "both range requests must reach the same provider account"); + assert!(requests[0].contains("range: bytes=0-9")); + assert!(requests[1].contains("range: bytes=5-14")); +} + +#[tokio::test] +async fn parallel_series_range_requests_keep_both_claims_active() { + const SERIES_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(SERIES_BODY, 2).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_501)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 43, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/series/1.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/series/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Series; + channel.cluster = XtreamCluster::Series; + channel.url = format!("http://{origin_addr}/series/1.ts").intern(); + channel + }; + + let first_headers = { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + headers + }; + let second_headers = { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=5-14")); + headers + }; + + // Open both range requests before consuming either body, so both claims are + // active at the same time. This exercises concurrent claims of one playback + // rather than a sequential request/consume cycle. + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &make_session("series-range-1"), + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &make_session("series-range-2"), + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(app_state.active_users.active_streams().await.len(), 2, "both range claims must be active concurrently"); + + let first_body = first.into_body().collect().await.expect("first series range body").to_bytes(); + assert_eq!(first_body.as_ref(), &SERIES_BODY[0..10]); + + let second_body = second.into_body().collect().await.expect("second series range body").to_bytes(); + assert_eq!(second_body.as_ref(), &SERIES_BODY[5..15]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 2, "both series range requests must reach the same provider account"); + assert!(requests[0].contains("range: bytes=0-9")); + assert!(requests[1].contains("range: bytes=5-14")); +} + +async fn spawn_load_test_origin(body: &'static [u8]) -> SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("load origin binds"); + let addr = listener.local_addr().expect("load origin address"); + let app = axum::Router::new().fallback(move || async move { + ([(axum::http::header::CONTENT_TYPE, "video/mp2t")], Bytes::from_static(body)) + }); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + addr +} + +fn load_test_latency_percentile(sorted: &[u64], p: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + let index = ((sorted.len() as f64 - 1.0) * p).round() as usize; + sorted[index.min(sorted.len() - 1)] +} + +fn validate_load_response(status: StatusCode, payload: &[u8], expected: &[u8]) -> Result<(), String> { + if status != StatusCode::OK { + return Err(format!("unexpected HTTP status {status}")); + } + if payload != expected { + return Err(format!("payload mismatch: expected {} bytes, received {}", expected.len(), payload.len())); + } + Ok(()) +} + +#[derive(Debug, Clone)] +struct SoakResourceSample { + elapsed_secs: f64, + rss_kib: Option, + provider_connections: usize, + lease_starting: usize, + lease_active: usize, + lease_idle: usize, + user_requests: usize, + user_sessions: usize, + shared_origins: usize, + shared_subscribers: usize, + shared_meters: usize, + running_tasks: usize, + completed: u64, + planned_aborted: u64, + unexpected_failed: u64, +} + +fn median_u64(values: &mut [u64]) -> Option { + if values.is_empty() { + return None; + } + values.sort_unstable(); + Some(values[values.len() / 2]) +} + +#[cfg(target_os = "linux")] +fn load_test_rss_kib() -> Option { + let Ok(content) = std::fs::read_to_string("/proc/self/status") else { + return None; + }; + for line in content.lines() { + if let Some(rest) = line.strip_prefix("VmRSS:") { + return rest.split_whitespace().next().and_then(|v| v.parse::().ok()); + } + } + None +} + +#[cfg(not(target_os = "linux"))] +fn load_test_rss_kib() -> Option { None } + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +#[ignore = "HTTP load test: cargo +stable test -p tuliprox --release --bin tuliprox -- --ignored --nocapture http_provider_lease_load_test"] +async fn http_provider_lease_load_test() { + const TS_PACKET: [u8; 188] = [0x47; 188]; + let origin_addr = spawn_load_test_origin(&TS_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + // Warm-up so allocator and connection pools are exercised before measurement. + for index in 0..32u16 { + let addr = SocketAddr::from(([127, 0, 0, 1], 45_000 + index)); + let fingerprint = create_test_fingerprint(addr); + let session = load_test_session(&input, origin_addr, &format!("warmup-{index}"), addr); + let channel = load_test_channel(&input, origin_addr); + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &load_test_user(&format!("warmup-user-{index}")), + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + let status = response.status(); + let body = response.into_body().collect().await.expect("warm-up body collects").to_bytes(); + validate_load_response(status, &body, &TS_PACKET).expect("warm-up response must match the fixture"); + } + + let baseline_rss_kib = load_test_rss_kib(); + eprintln!("HTTP provider lease load test (unlimited provider, real HTTP body, 5 repetitions per concurrency)"); + for concurrency in [1usize, 5, 50, 200] { + let rounds_per_task = 20; + let repetitions = 5; + let mut all_samples = Vec::<(u64, u64)>::new(); + let mut total_ops = 0u64; + let mut total_elapsed_secs = 0f64; + for rep in 0..repetitions { + let samples = Arc::new(std::sync::Mutex::new(Vec::<(u64, u64)>::new())); + let started = std::time::Instant::now(); + let mut tasks = tokio::task::JoinSet::new(); + for task_index in 0..concurrency { + let app = Arc::clone(&app); + let input = Arc::clone(&input); + let samples = Arc::clone(&samples); + let username = format!("load-user-{rep}-{task_index}"); + tasks.spawn(async move { + for round in 0..rounds_per_task { + let addr = SocketAddr::from(([127, 0, 0, 1], 46_000 + (task_index as u16))); + let fingerprint = create_test_fingerprint(addr); + let token = format!("load-{rep}-{task_index}-{round}"); + let session = load_test_session(&input, origin_addr, &token, addr); + let channel = load_test_channel(&input, origin_addr); + let user = load_test_user(&username); + + let acquire_at = std::time::Instant::now(); + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + let acquire_us = acquire_at.elapsed().as_micros() as u64; + + let status = response.status(); + let body = response.into_body(); + let body_at = std::time::Instant::now(); + let collected = body.collect().await.expect("load body collects").to_bytes(); + validate_load_response(status, &collected, &TS_PACKET) + .expect("load response must match the TS packet"); + let body_us = body_at.elapsed().as_micros() as u64; + + samples.lock().unwrap().push((acquire_us, body_us)); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.expect("load task must not fail"); + } + + total_elapsed_secs += started.elapsed().as_secs_f64(); + let rep_samples = Arc::try_unwrap(samples).expect("samples unique").into_inner().unwrap(); + total_ops += rep_samples.len() as u64; + all_samples.extend(rep_samples); + } + + let mut acquire = Vec::with_capacity(all_samples.len()); + let mut body = Vec::with_capacity(all_samples.len()); + for (a, b) in all_samples { + acquire.push(a); + body.push(b); + } + acquire.sort_unstable(); + body.sort_unstable(); + let throughput = total_ops as f64 / total_elapsed_secs.max(f64::EPSILON); + let byte_throughput = total_ops as f64 * TS_PACKET.len() as f64 / total_elapsed_secs.max(f64::EPSILON); + let rss_delta_kib = load_test_rss_kib() + .zip(baseline_rss_kib) + .map(|(rss, baseline)| rss.saturating_sub(baseline).to_string()) + .unwrap_or_else(|| "unsupported".to_string()); + eprintln!( + "concurrency={concurrency:>3} ops={total_ops:>5} throughput={throughput:>9.1} ops/s {byte_throughput:>11.1} bytes/s | acquire p50/p95/p99={}/{}/{}us | body p50/p95/p99={}/{}/{}us | rss_delta_kib={rss_delta_kib}", + load_test_latency_percentile(&acquire, 0.50), + load_test_latency_percentile(&acquire, 0.95), + load_test_latency_percentile(&acquire, 0.99), + load_test_latency_percentile(&body, 0.50), + load_test_latency_percentile(&body, 0.95), + load_test_latency_percentile(&body, 0.99), + ); + let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; + let shared_meters = app.shared_stream_manager.meter_count(); + let leases = app.active_provider.provider_lease_usage(&input.name); + eprintln!( + "[load resource] concurrency={concurrency} conns={} starting={} active={} idle={} user_requests={user_requests} user_sessions={user_sessions} shared_origins={shared_origins} shared_subscribers={shared_subscribers} shared_meters={shared_meters}", + app.active_provider.get_provider_connections_count(), + leases.starting, + leases.active, + leases.idle, + ); + } + + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let leases = app.active_provider.provider_lease_usage(&input.name); + let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; + let shared_meters = app.shared_stream_manager.meter_count(); + if app.active_provider.get_provider_connections_count() == 0 + && leases.total() == 0 + && user_requests == 0 + && user_sessions == 0 + && shared_origins == 0 + && shared_subscribers == 0 + && shared_meters == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("load resources must drain to baseline"); + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "provider slots must return to baseline"); + let usage = app.active_provider.provider_lease_usage(&input.name); + assert_eq!(usage.total(), 0, "lease table must return to baseline after HTTP churn"); +} + +fn load_test_user(username: &str) -> ProxyUserCredentials { + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user +} + +fn load_test_session(input: &ConfigInput, origin_addr: SocketAddr, token: &str, addr: SocketAddr) -> UserSession { + UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/live/42.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr, + socket_bound: false, + active_addrs: vec![addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + } +} + +fn load_test_channel(input: &ConfigInput, origin_addr: SocketAddr) -> StreamChannel { + let mut channel = create_test_live_channel(&format!("http://{origin_addr}/live/42.ts")); + channel.provider_id = u32::from(input.id); + channel.input_name = Arc::clone(&input.name); + channel.url = format!("http://{origin_addr}/live/42.ts").intern(); + channel +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +#[ignore = "30-minute soak: TULIPROX_SOAK_SECS=60 cargo +stable test -p tuliprox --release --bin tuliprox -- --ignored --nocapture http_provider_lease_soak_test"] +async fn http_provider_lease_soak_test() { + let duration_secs: u64 = std::env::var("TULIPROX_SOAK_SECS").ok().and_then(|v| v.parse().ok()).unwrap_or(1800); + const SOAK_PACKET: [u8; 188] = [0x47; 188]; + let origin_addr = spawn_load_test_origin(&SOAK_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + let baseline_rss_kib = load_test_rss_kib(); + let completed = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let aborted = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let unexpected_failed = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let peak_connections = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let running_tasks = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let resource_samples = Arc::new(std::sync::Mutex::new(Vec::::new())); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(duration_secs); + let sampler_stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let sampler_stop_task = Arc::clone(&sampler_stop); + let app_sampler = Arc::clone(&app); + let sampler_input_name = Arc::clone(&input.name); + let sampler_completed = Arc::clone(&completed); + let sampler_aborted = Arc::clone(&aborted); + let sampler_failed = Arc::clone(&unexpected_failed); + let sampler_running = Arc::clone(&running_tasks); + let sampler_samples = Arc::clone(&resource_samples); + let sampler_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); + let start_time = std::time::Instant::now(); + while !sampler_stop_task.load(std::sync::atomic::Ordering::Relaxed) { + interval.tick().await; + let rss = load_test_rss_kib().map_or_else(|| "unsupported".to_string(), |rss| rss.to_string()); + let conns = app_sampler.active_provider.get_provider_connections_count(); + let leases = app_sampler.active_provider.provider_lease_usage(&sampler_input_name); + let (user_requests, user_sessions) = app_sampler.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app_sampler.shared_stream_manager.resource_counts().await; + let shared_meters = app_sampler.shared_stream_manager.meter_count(); + let sample = SoakResourceSample { + elapsed_secs: start_time.elapsed().as_secs_f64(), + rss_kib: load_test_rss_kib(), + provider_connections: conns, + lease_starting: leases.starting, + lease_active: leases.active, + lease_idle: leases.idle, + user_requests, + user_sessions, + shared_origins, + shared_subscribers, + shared_meters, + running_tasks: sampler_running.load(std::sync::atomic::Ordering::Relaxed), + completed: sampler_completed.load(std::sync::atomic::Ordering::Relaxed), + planned_aborted: sampler_aborted.load(std::sync::atomic::Ordering::Relaxed), + unexpected_failed: sampler_failed.load(std::sync::atomic::Ordering::Relaxed), + }; + eprintln!( + "[soak time_series] t={:4.1}s rss_kib={} conns={} starting={} active={} idle={} user_requests={} user_sessions={} shared_origins={} shared_subscribers={} shared_meters={} running_tasks={} completed={} planned_aborted={} unexpected_failed={}", + sample.elapsed_secs, + rss, + sample.provider_connections, + sample.lease_starting, + sample.lease_active, + sample.lease_idle, + sample.user_requests, + sample.user_sessions, + sample.shared_origins, + sample.shared_subscribers, + sample.shared_meters, + sample.running_tasks, + sample.completed, + sample.planned_aborted, + sample.unexpected_failed, + ); + sampler_samples.lock().unwrap().push(sample); + } + }); + + let mut tasks = tokio::task::JoinSet::new(); + for task_index in 0..16u16 { + let app = Arc::clone(&app); + let input = Arc::clone(&input); + let completed = Arc::clone(&completed); + let aborted = Arc::clone(&aborted); + let unexpected_failed = Arc::clone(&unexpected_failed); + let peak_connections = Arc::clone(&peak_connections); + let running_tasks = Arc::clone(&running_tasks); + tasks.spawn(async move { + running_tasks.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let result = async { + let addr = SocketAddr::from(([127, 0, 0, 1], 47_000 + task_index)); + let fingerprint = create_test_fingerprint(addr); + let username = format!("soak-user-{task_index}"); + let user = load_test_user(&username); + let token = format!("soak-reconnect-family-{task_index}"); + let mut round = 0u64; + while std::time::Instant::now() < deadline { + round += 1; + let session = load_test_session(&input, origin_addr, &token, addr); + let channel = load_test_channel(&input, origin_addr); + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 2, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let status = response.status(); + if status != StatusCode::OK { + unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Err(format!("soak response status was {status}")); + } + // Alternate full playback (collect) and abort (drop without consuming). + if round % 2 == 0 { + let collected = match response.into_body().collect().await { + Ok(body) => body.to_bytes(), + Err(err) => { + unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Err(format!("soak body collection failed: {err}")); + } + }; + if let Err(err) = validate_load_response(status, &collected, &SOAK_PACKET) { + unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Err(err); + } + completed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } else { + drop(response); + aborted.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + let current = app.active_provider.get_provider_connections_count(); + peak_connections.fetch_max(current, std::sync::atomic::Ordering::Relaxed); + } + Ok::<(), String>(()) + } + .await; + running_tasks.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + result + }); + } + while let Some(result) = tasks.join_next().await { + result.expect("soak task must join").unwrap_or_else(|err| panic!("soak task failed: {err}")); + } + + sampler_stop.store(true, std::sync::atomic::Ordering::Relaxed); + let _ = sampler_task.await; + + let completed = completed.load(std::sync::atomic::Ordering::Relaxed); + let aborted = aborted.load(std::sync::atomic::Ordering::Relaxed); + let peak = peak_connections.load(std::sync::atomic::Ordering::Relaxed); + let unexpected_failed = unexpected_failed.load(std::sync::atomic::Ordering::Relaxed); + let rss_delta_kib = load_test_rss_kib() + .zip(baseline_rss_kib) + .map(|(rss, baseline)| rss.saturating_sub(baseline).to_string()) + .unwrap_or_else(|| "unsupported".to_string()); + eprintln!( + "soak duration={duration_secs}s completed={completed} planned_aborted={aborted} unexpected_failed={unexpected_failed} peak_connections={peak} rss_delta_kib={rss_delta_kib}" + ); + + let samples = Arc::try_unwrap(resource_samples).expect("resource samples unique").into_inner().unwrap(); + assert!(!samples.is_empty(), "soak must record at least one resource sample"); + let third = (samples.len() / 3).max(1); + let middle_start = third.min(samples.len()); + let middle_end = (third.saturating_mul(2)).min(samples.len()); + let last_start = middle_end.min(samples.len()); + let mut middle_rss = + samples[middle_start..middle_end].iter().filter_map(|sample| sample.rss_kib).collect::>(); + let mut last_rss = samples[last_start..].iter().filter_map(|sample| sample.rss_kib).collect::>(); + match (median_u64(&mut middle_rss), median_u64(&mut last_rss)) { + (Some(middle), Some(last)) => { + let delta = i128::from(last) - i128::from(middle); + eprintln!("soak rss rolling-median middle_third={middle}KiB last_third={last}KiB delta={delta}KiB"); + } + _ => eprintln!("soak rss rolling-median unsupported on this platform"), + } + + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let leases = app.active_provider.provider_lease_usage(&input.name); + let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; + let shared_meters = app.shared_stream_manager.meter_count(); + if app.active_provider.get_provider_connections_count() == 0 + && leases.total() == 0 + && user_requests == 0 + && user_sessions == 0 + && shared_origins == 0 + && shared_subscribers == 0 + && shared_meters == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("soak resources must drain to baseline"); + + assert_eq!( + app.active_provider.get_provider_connections_count(), + 0, + "provider slots must return to baseline after soak" + ); + let usage = app.active_provider.provider_lease_usage(&input.name); + assert_eq!(usage.total(), 0, "lease table must return to baseline after soak"); + assert_eq!(unexpected_failed, 0, "soak must not hide failed operations"); + assert!(peak <= 16, "active connections must stay bounded by the churn workers, observed {peak}"); +} + +#[tokio::test] +async fn direct_ts_eof_before_first_byte_releases_slot_without_idle_lease() { + let (origin_addr, origin_task) = spawn_controlled_fake_origin(FakeOriginMode::EmptyBody, 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_601)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_ts_eof".to_string(); + + let session = UserSession { + token: "session-eof".to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/live/42.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + + let channel = load_test_channel(&input, origin_addr); + + let response = force_provider_stream_response( + &fingerprint, + &app_state, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let body_bytes = response.into_body().collect().await.expect("empty body collects").to_bytes(); + assert!(body_bytes.is_empty(), "expected 0 payload bytes on immediate EOF"); + + let requests = origin_task.await.expect("origin finishes"); + assert_eq!(requests.len(), 1); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 + && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "physical slot must be released"); + let lease_usage = app_state.active_provider.provider_lease_usage(&input.name); + assert_eq!(lease_usage.starting, 0, "no starting leases remaining"); + assert_eq!(lease_usage.active, 0, "no active leases remaining"); + assert_eq!(lease_usage.idle, 0, "unconfirmed stream must NOT transition to an idle lease"); + assert_eq!(lease_usage.total(), 0, "total leases must be 0"); + assert_eq!(app_state.active_users.active_streams().await.len(), 0, "active user stream must be cleaned up"); +} + +#[tokio::test] +async fn direct_ts_abort_while_waiting_for_first_byte_releases_exact_request() { + let notify = Arc::new(tokio::sync::Notify::new()); + let (origin_addr, origin_task) = + spawn_controlled_fake_origin(FakeOriginMode::BlockFirstChunk(Arc::clone(¬ify)), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_602)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_ts_abort".to_string(); + + let session = UserSession { + token: "session-abort-before-first-byte".to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/live/42.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + + let channel = load_test_channel(&input, origin_addr); + + let response = force_provider_stream_response( + &fingerprint, + &app_state, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + drop(response); + notify.notify_waiters(); + + let _ = origin_task.await; + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 + && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!( + app_state.active_provider.get_provider_connections_count(), + 0, + "physical slot must be released on abort" + ); + let lease_usage = app_state.active_provider.provider_lease_usage(&input.name); + assert_eq!(lease_usage.total(), 0, "no residual lease claim after unconfirmed abort"); + assert_eq!(app_state.active_users.active_streams().await.len(), 0, "active user stream must be cleaned up"); +} + +#[tokio::test] +async fn provider_error_after_first_byte_finishes_only_matching_request() { + const TS_CHUNK_188: &[u8] = &[0x47; 188]; + const SIBLING_CHUNK: &[u8] = &[0x47; 376]; + + let (origin_fail_addr, origin_fail_task) = + spawn_controlled_fake_origin(FakeOriginMode::ErrorAfterFirstChunk(TS_CHUNK_188.to_vec()), 1).await; + let (origin_ok_addr, origin_ok_task) = + spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(SIBLING_CHUNK.to_vec()), 1).await; + + let input_fail = Arc::new(ConfigInput { + id: 1, + name: "provider_fail".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_fail_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let input_ok = Arc::new(ConfigInput { + id: 2, + name: "provider_ok".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_ok_addr}"), + enabled: true, + priority: 1, + max_connections: 2, + ..ConfigInput::default() + }); + + let mut config = create_test_provider_app_config(); + config.sources = Arc::new(ArcSwap::from_pointee(SourcesConfig { + inputs: vec![Arc::clone(&input_fail), Arc::clone(&input_ok)], + ..SourcesConfig::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let addr_fail = SocketAddr::from(([127, 0, 0, 1], 55_603)); + let addr_ok = SocketAddr::from(([127, 0, 0, 1], 55_604)); + + let user = load_test_user("viewer_multistream"); + let session_fail = load_test_session(&input_fail, origin_fail_addr, "session-fail", addr_fail); + let session_ok = load_test_session(&input_ok, origin_ok_addr, "session-ok", addr_ok); + + let channel_fail = load_test_channel(&input_fail, origin_fail_addr); + let channel_ok = load_test_channel(&input_ok, origin_ok_addr); + + let resp_fail = force_provider_stream_response( + &create_test_fingerprint(addr_fail), + &app_state, + &session_fail, + channel_fail, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input_fail, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let resp_ok = force_provider_stream_response( + &create_test_fingerprint(addr_ok), + &app_state, + &session_ok, + channel_ok, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input_ok, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp_fail.status(), StatusCode::OK); + assert_eq!(resp_ok.status(), StatusCode::OK); + assert_eq!( + app_state.active_provider.get_provider_connections_count(), + 2, + "both providers have an active connection" + ); + + let mut fail_body = resp_fail.into_body(); + let first_frame = fail_body.frame().await.expect("first frame exists").expect("first frame is valid"); + let first_chunk = first_frame.into_data().expect("data chunk exists"); + assert_eq!(first_chunk.as_ref(), TS_CHUNK_188, "first byte was confirmed with exact chunk bytes"); + + let second_frame_res = fail_body.frame().await; + assert!( + second_frame_res.is_none() || second_frame_res.as_ref().is_some_and(Result::is_err), + "stream must terminate or error after first chunk" + ); + drop(fail_body); + + for _ in 0..50 { + if app_state.active_provider.provider_lease_usage(&input_fail.name).total() == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!( + app_state.active_provider.provider_lease_usage(&input_fail.name).total(), + 0, + "failed provider leases cleaned up" + ); + assert_eq!( + app_state.active_provider.provider_lease_usage(&input_ok.name).total(), + 1, + "sibling provider lease remains intact" + ); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1, "sibling connection still active"); + + let ok_bytes = resp_ok.into_body().collect().await.expect("sibling body collects").to_bytes(); + assert_eq!(ok_bytes.as_ref(), SIBLING_CHUNK); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "all slots returned to baseline"); + let _ = origin_fail_task.await; + let _ = origin_ok_task.await; +} + +#[tokio::test] +async fn no_first_byte_failure_allows_next_request_to_prefer_provider_a() { + const TS_CHUNK: &[u8] = &[0x47; 188]; + let listener_a = TcpListener::bind("127.0.0.1:0").await.expect("origin A binds"); + let origin_a_addr = listener_a.local_addr().expect("origin A address"); + let task_a = tokio::spawn(async move { + let mut requests = Vec::new(); + for round in 0..2 { + let Ok((mut socket, _)) = listener_a.accept().await else { break }; + let mut req = Vec::new(); + while !req.windows(4).any(|w| w == b"\r\n\r\n") { + let mut chunk = [0u8; 512]; + let Ok(read) = socket.read(&mut chunk).await else { break }; + if read == 0 { + break; + } + req.extend_from_slice(&chunk[..read]); + } + requests.push(String::from_utf8_lossy(&req).to_ascii_lowercase()); + if round == 0 { + let head = + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } else { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + TS_CHUNK.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(TS_CHUNK).await; + let _ = socket.shutdown().await; + } + } + requests + }); + + let (origin_b_addr, _task_b) = spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(TS_CHUNK.to_vec()), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_A".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_a_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_B".intern(), + url: format!("http://{origin_b_addr}"), + username: None, + password: None, + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_605)); + let fingerprint = create_test_fingerprint(client_addr); + let user = load_test_user("viewer_prefer_a"); + + let session_1 = load_test_session(&input, origin_a_addr, "session-1", client_addr); + let channel_1 = load_test_channel(&input, origin_a_addr); + + let resp_1 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_1, + channel_1, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp_1.status(), StatusCode::OK); + let b1 = resp_1.into_body().collect().await.expect("body 1 collects").to_bytes(); + assert!(b1.is_empty(), "request 1 had early EOF"); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 + && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "slot on A must be free"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0, "no phantom lease on A"); + + let session_2 = load_test_session(&input, origin_a_addr, "session-2", client_addr); + let channel_2 = load_test_channel(&input, origin_a_addr); + + let resp_2 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_2, + channel_2, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp_2.status(), StatusCode::OK); + let b2 = resp_2.into_body().collect().await.expect("body 2 collects").to_bytes(); + assert_eq!(b2.as_ref(), TS_CHUNK, "request 2 successfully used Provider A and read payload"); + + let reqs_a = task_a.await.expect("task A completed"); + assert_eq!(reqs_a.len(), 2, "both requests 1 and 2 were routed to Provider A!"); +} + +#[tokio::test] +async fn parallel_vod_abort_preserves_sibling_claim_and_provider_stickiness() { + const VOD_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(VOD_BODY, 3).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_701)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_vod_parallel".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/movie/1.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/movie/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = format!("http://{origin_addr}/movie/1.ts").intern(); + channel + }; + + let build_request = |range: &'static str, token: &'static str| { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static(range)); + let session = make_session(token); + (headers, session) + }; + + let (first_headers, first_session) = build_request("bytes=0-9", "vod-playback"); + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &first_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + + let (second_headers, second_session) = build_request("bytes=5-14", "vod-playback"); + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &second_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "affine claims share one playback row"); + assert_eq!(app_state.active_users.playback_resource_counts().await.0, 2, "both request claims active"); + + drop(first); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_users.playback_resource_counts().await.0 != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("aborted VOD body must release only its request claim"); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "sibling playback row remains active"); + + let second_body = second.into_body().collect().await.expect("second body collects").to_bytes(); + assert_eq!(second_body.as_ref(), &VOD_BODY[5..15]); + + let (third_headers, third_session) = build_request("bytes=10-19", "vod-playback"); + let third = force_provider_stream_response( + &fingerprint, + &app_state, + &third_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &third_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(third.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + third.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), + Some("bytes 10-19/20") + ); + let third_body = third.into_body().collect().await.expect("third body collects").to_bytes(); + assert_eq!(third_body.as_ref(), &VOD_BODY[10..20]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 3, "all 3 range requests must reach the same provider account"); + assert!(requests[0].contains("range: bytes=0-9")); + assert!(requests[1].contains("range: bytes=5-14")); + assert!(requests[2].contains("range: bytes=10-19")); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("VOD provider connections must drain"); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + app_state.active_provider.clear_provider_reservation("vod-playback"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); +} + +#[tokio::test] +async fn parallel_series_abort_then_seek_reuses_account_without_stale_claim() { + const SERIES_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(SERIES_BODY, 3).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_702)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_series_parallel".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 43, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/series/1.ts").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/series/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Series; + channel.cluster = XtreamCluster::Series; + channel.url = format!("http://{origin_addr}/series/1.ts").intern(); + channel + }; + + let build_request = |range: &'static str, token: &'static str| { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static(range)); + let session = make_session(token); + (headers, session) + }; + + let (first_headers, first_session) = build_request("bytes=0-9", "series-playback"); + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &first_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + + let (second_headers, second_session) = build_request("bytes=5-14", "series-playback"); + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &second_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "affine claims share one playback row"); + assert_eq!(app_state.active_users.playback_resource_counts().await.0, 2, "both request claims active"); + + drop(first); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_users.playback_resource_counts().await.0 != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("aborted Series body must release only its request claim"); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "sibling playback row remains active"); + + let second_body = second.into_body().collect().await.expect("second body collects").to_bytes(); + assert_eq!(second_body.as_ref(), &SERIES_BODY[5..15]); + + let (third_headers, third_session) = build_request("bytes=12-19", "series-playback"); + let third = force_provider_stream_response( + &fingerprint, + &app_state, + &third_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &third_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(third.status(), StatusCode::PARTIAL_CONTENT); + let third_body = third.into_body().collect().await.expect("third body collects").to_bytes(); + assert_eq!(third_body.as_ref(), &SERIES_BODY[12..20]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 3, "all series requests reach same provider"); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("series provider connections must drain"); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + app_state.active_provider.clear_provider_reservation("series-playback"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); +} + +#[tokio::test] +async fn stale_direct_cleanup_after_rebind_preserves_successor_request() { + let app_state = create_test_provider_app_state(); + let provider_name: Arc = Arc::from("provider_1"); + let owner = "test-rebind-owner"; + let addr_x = SocketAddr::from(([127, 0, 0, 1], 55_631)); + let addr_y = SocketAddr::from(([127, 0, 0, 1], 55_632)); + + let req_x = tuliprox_core::model::PlaybackRequestId::from_raw(101); + let lease_ref_x = + tuliprox_session::PlaybackLeaseRef { owner, kind: tuliprox_core::model::PlaybackKind::Vod, request_id: req_x }; + let handle_x = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_x, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_x), + ) + .expect("request X acquires provider"); + let stale_tag = handle_x.binding_tag.expect("request X binding tag"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_x, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_x); + app_state.active_provider.release_handle(&handle_x); + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + assert_eq!(app_state.active_provider.provider_lease_usage(&provider_name).idle, 1); + + let req_y = tuliprox_core::model::PlaybackRequestId::from_raw(102); + let lease_ref_y = + tuliprox_session::PlaybackLeaseRef { owner, kind: tuliprox_core::model::PlaybackKind::Vod, request_id: req_y }; + let handle_y = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_y, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_y), + ) + .expect("request Y reacquires provider"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_y, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_y); + + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + + let usage_after_stale_finish = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(usage_after_stale_finish.active, 1, "successor request Y must keep the active lease alive"); + + app_state.active_provider.clear_identified_provider_reservation(owner, &provider_name, Some(stale_tag)); + + let usage_after_stale_tag = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(usage_after_stale_tag.active, 1, "stale clear tag must not erase active successor lease"); + + app_state.active_provider.release_handle(&handle_y); + app_state.active_provider.finish_identified_playback_request( + owner, + req_y, + tuliprox_core::model::PlaybackRequestOutcome::ServerShutdown, + ); + + let final_usage = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(final_usage.total(), 0, "request Y has finished"); +} + +#[tokio::test] +async fn stale_hls_cleanup_after_rebind_preserves_current_origin_and_segments() { + let app_state = create_test_provider_app_state(); + let provider_name: Arc = Arc::from("provider_1"); + let owner = "test-hls-rebind-owner"; + let addr_x = SocketAddr::from(([127, 0, 0, 1], 55_633)); + let addr_y = SocketAddr::from(([127, 0, 0, 1], 55_634)); + + let req_x = tuliprox_core::model::PlaybackRequestId::from_raw(201); + let lease_ref_x = tuliprox_session::PlaybackLeaseRef { + owner, + kind: tuliprox_core::model::PlaybackKind::LiveHls, + request_id: req_x, + }; + let handle_x = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_x, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_x), + ) + .expect("request X acquires HLS provider"); + let stale_tag = handle_x.binding_tag.expect("request X binding tag"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_x, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_x); + app_state.active_provider.release_handle(&handle_x); + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + + let req_y = tuliprox_core::model::PlaybackRequestId::from_raw(202); + let lease_ref_y = tuliprox_session::PlaybackLeaseRef { + owner, + kind: tuliprox_core::model::PlaybackKind::LiveHls, + request_id: req_y, + }; + let handle_y = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_y, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_y), + ) + .expect("request Y reacquires HLS provider"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_y, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_y); + + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + + let usage = app_state.active_provider.provider_lease_usage(&provider_name); + assert!(usage.active >= 1 || usage.idle >= 1, "active/reconnect lease remains for successor"); + + app_state.active_provider.clear_identified_provider_reservation(owner, &provider_name, Some(stale_tag)); + + let usage2 = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(usage2.total(), usage.total(), "stale clear tag must not drop the HLS lease"); + + app_state.active_provider.release_handle(&handle_y); + app_state.active_provider.finish_identified_playback_request( + owner, + req_y, + tuliprox_core::model::PlaybackRequestOutcome::ServerShutdown, + ); + assert_eq!(app_state.active_provider.provider_lease_usage(&provider_name).total(), 0); +} + +#[tokio::test] +async fn shared_proxy_clients_keep_independent_bodies_and_cleanup() { + let mut target = create_test_shared_target(); + target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; + let target = Arc::new(target); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("shared origin binds"); + let origin_addr = listener.local_addr().expect("shared origin addr"); + let app = axum::Router::new().fallback(|| async { + let (tx, rx) = tokio::sync::mpsc::channel::>(16); + tokio::spawn(async move { + for i in 0..10u8 { + let _ = tx.send(Ok(Bytes::from(vec![i; 188]))).await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + }); + let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx); + axum::response::Response::builder() + .header(axum::http::header::CONTENT_TYPE, "video/mp2t") + .body(axum::body::Body::from_stream(body_stream)) + .unwrap() + }); + let origin_task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_shared_p".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + priority: 0, + max_connections: 5, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let proxy_addr = SocketAddr::from(([127, 0, 0, 1], 55_190)); + let fp1 = Fingerprint::new("client1|ua1".to_string(), "127.0.0.1".to_string(), proxy_addr); + let fp2 = Fingerprint::new("client2|ua2".to_string(), "127.0.0.1".to_string(), proxy_addr); + + let user1 = load_test_user("user_shared_1"); + let user2 = load_test_user("user_shared_2"); + let stream_url = format!("http://{origin_addr}/live/user/pass/shared.ts"); + let mut channel1 = load_test_channel(&input, origin_addr); + channel1.url = stream_url.as_str().intern(); + let mut channel2 = load_test_channel(&input, origin_addr); + channel2.url = stream_url.as_str().intern(); + + let resp1 = stream_response( + &fp1, + &app_state, + "shared-session-1", + None, + channel1, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user1, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + assert_eq!(resp1.status(), StatusCode::OK); + + let resp2 = stream_response( + &fp2, + &app_state, + "shared-session-2", + None, + channel2, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user2, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + assert_eq!(resp2.status(), StatusCode::OK); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1, "only 1 shared origin connection"); + + drop(resp1); + + let body2 = resp2.into_body().collect().await.expect("client 2 body collects").to_bytes(); + assert!(!body2.is_empty(), "client 2 receives stream bytes after client 1 dropped"); + drop(body2); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + origin_task.abort(); +} + +#[tokio::test] +async fn shared_admission_rejection_does_not_leak_meter_registration() { + let mut target = create_test_shared_target(); + target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; + let target = Arc::new(target); + + let origin_addr = spawn_load_test_origin(b"adm-test-ts-chunk").await; + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_shared_adm".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + priority: 0, + max_connections: 5, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + config.config = Arc::new(ArcSwap::from_pointee(Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(crate::model::StreamConfig { metrics_enabled: true, ..crate::model::StreamConfig::default() }), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let cleanup_tx = app_state.connection_manager.cleanup_tx(); + let mut permits = Vec::new(); + while let Ok(permit) = cleanup_tx.clone().try_reserve_owned() { + permits.push(permit); + } + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_195)); + let fp = create_test_fingerprint(client_addr); + let user = load_test_user("user_adm_fail"); + let stream_url = format!("http://{origin_addr}/live/user/pass/adm.ts"); + let mut channel = load_test_channel(&input, origin_addr); + channel.url = stream_url.as_str().intern(); + + let resp = stream_response( + &fp, + &app_state, + "shared-session-adm", + None, + channel, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "shared admission failure must yield 503 Service Unavailable" + ); + assert_eq!( + app_state.active_provider.get_provider_connections_count(), + 0, + "admission failure must NOT fall through to opening a provider connection" + ); + assert_eq!(app_state.shared_stream_manager.meter_count(), 0, "rejected admission must not retain a meter ID"); + + drop(permits); +} + +#[tokio::test] +async fn shared_start_abort_does_not_leak_meter_registration() { + let mut target = create_test_shared_target(); + target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; + let target = Arc::new(target); + + let notify = Arc::new(tokio::sync::Notify::new()); + let (origin_addr, origin_task) = + spawn_controlled_fake_origin(FakeOriginMode::BlockFirstChunk(Arc::clone(¬ify)), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_shared_abort".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + config.config = Arc::new(ArcSwap::from_pointee(Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(crate::model::StreamConfig { metrics_enabled: true, ..crate::model::StreamConfig::default() }), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_196)); + let fp = create_test_fingerprint(client_addr); + let user = load_test_user("user_shared_abort"); + let stream_url = format!("http://{origin_addr}/live/user/pass/shared_abort.ts"); + let mut channel = load_test_channel(&input, origin_addr); + channel.url = stream_url.as_str().intern(); + + let resp = stream_response( + &fp, + &app_state, + "shared-session-abort", + None, + channel, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + + assert_eq!(resp.status(), StatusCode::OK); + drop(resp); + notify.notify_waiters(); + let _ = origin_task.await; + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 + || app_state.active_provider.provider_lease_usage(&input.name).total() != 0 + || !app_state.active_users.active_streams().await.is_empty() + || app_state.shared_stream_manager.meter_count() != 0 + { + tokio::task::yield_now().await; + } + }) + .await + .expect("shared abort resources must return to baseline"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "provider slot released"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0, "provider request released"); + assert_eq!(app_state.active_users.active_streams().await.len(), 0, "user claim released"); + assert_eq!(app_state.shared_stream_manager.meter_count(), 0, "aborted origin meter registration released"); +} + +#[tokio::test] +async fn catchup_abort_seek_and_window_change_preserve_correct_affinity() { + const CATCHUP_BODY_A: &[u8] = b"catchup-origin-a-archive-payload"; + const CATCHUP_BODY_B: &[u8] = b"catchup-origin-b-archive-payload"; + + let (origin_a, task_a) = spawn_range_aware_test_origin(CATCHUP_BODY_A, 2).await; + let (origin_b, task_b) = spawn_range_aware_test_origin(CATCHUP_BODY_B, 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_a}"), + enabled: true, + priority: 0, + max_connections: 2, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: format!("http://{origin_b}"), + username: None, + password: None, + priority: 1, + max_connections: 2, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_620)); + let fingerprint = create_test_fingerprint(client_addr); + let user = load_test_user("viewer_catchup"); + + let make_catchup_session = |token: &str, window: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 41, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_a}/timeshift/1.ts?window={window}").intern(), + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + + let make_catchup_channel = |window: &str| { + let mut channel = create_test_local_channel(&format!("http://{origin_a}/timeshift/1.ts?window={window}")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Catchup; + channel.cluster = XtreamCluster::Live; + channel.url = format!("http://{origin_a}/timeshift/1.ts?window={window}").intern(); + channel + }; + + let session_w1_1 = make_catchup_session("catchup-w1", "w1"); + let session_w1_2 = make_catchup_session("catchup-w1", "w1"); + + let mut h1 = HeaderMap::new(); + h1.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + let mut h2 = HeaderMap::new(); + h2.insert(header::RANGE, HeaderValue::from_static("bytes=5-14")); + + let resp1 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_w1_1, + make_catchup_channel("w1"), + ForceStreamRequestContext { + req_headers: &h1, + input: &input, + user: &user, + session_reservation_ttl_secs: 10, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let resp2 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_w1_2, + make_catchup_channel("w1"), + ForceStreamRequestContext { + req_headers: &h2, + input: &input, + user: &user, + session_reservation_ttl_secs: 10, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp1.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(resp2.status(), StatusCode::PARTIAL_CONTENT); + + let mut session_w2 = make_catchup_session("catchup-w2", "w2"); + session_w2.provider = "provider_2".intern(); + session_w2.stream_url = format!("http://{origin_b}/timeshift/1.ts?window=w2").intern(); + let mut channel_w2 = make_catchup_channel("w2"); + channel_w2.url = Arc::clone(&session_w2.stream_url); + let mut h_w2 = HeaderMap::new(); + h_w2.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + let resp_w2 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_w2, + channel_w2, + ForceStreamRequestContext { + req_headers: &h_w2, + input: &input, + user: &user, + session_reservation_ttl_secs: 10, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(resp_w2.status(), StatusCode::PARTIAL_CONTENT); + let bytes_w2 = resp_w2.into_body().collect().await.expect("window w2 body").to_bytes(); + assert_eq!(bytes_w2.as_ref(), &CATCHUP_BODY_B[..10], "independent window uses available provider B"); + + drop(resp1); + let bytes2 = resp2.into_body().collect().await.expect("resp2 body").to_bytes(); + assert_eq!(bytes2.as_ref(), &CATCHUP_BODY_A[5..15]); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("parallel catchup requests must release their physical connections"); + + let reqs_a = task_a.await.expect("task A completes"); + assert_eq!(reqs_a.len(), 2, "both requests in window w1 stayed on provider A"); + let reqs_b = task_b.await.expect("task B completes"); + assert_eq!(reqs_b.len(), 1, "the independent window used provider B while A was full"); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("catchup provider connections must drain"); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + app_state.active_provider.clear_provider_reservation("catchup-w1"); + app_state.active_provider.clear_provider_reservation("catchup-w2"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); + assert_eq!(app_state.active_provider.provider_lease_usage(&"provider_2".intern()).total(), 0); +} + +#[tokio::test] +async fn dash_stream_request_remains_redirect_and_preserves_provider_affinity() { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_dash".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-dash.example".to_string(), + username: Some("dash-user".to_string()), + password: Some("dash-pass".to_string()), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let target = Arc::new(ConfigTarget { + id: 1, + name: "target_dash".to_string(), + enabled: true, + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }); + + let mut app_cfg = create_test_provider_app_config(); + app_cfg.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); + + let pli = PlaylistItem { + header: PlaylistItemHeader { + id: "1".intern(), + name: "DASH Live".intern(), + group: "Live".intern(), + title: "DASH Live".intern(), + url: "http://provider-dash.example/live/dash-user/dash-pass/1.mpd".intern(), + item_type: PlaylistItemType::LiveDash, + xtream_cluster: XtreamCluster::Live, + virtual_id: shared::model::VirtualId(1), + ..PlaylistItemHeader::default() + }, + }; + + let mut user = load_test_user("dash_viewer"); + user.proxy = ProxyType::Reverse(None); + for target_type in [TargetType::M3u, TargetType::Xtream] { + let redirect_params = RedirectParams { + item: &pli, + provider_id: pli.get_provider_id(), + cluster: XtreamCluster::Live, + target_type, + target: &target, + input: &input, + user: &user, + stream_ext: Some(shared::defaults::DASH_EXT), + req_context: crate::api::endpoints::xtream_api::ApiStreamContext::Live, + action_path: "", + }; + + let resp = redirect_response(&app_state, &redirect_params) + .await + .expect("DASH must return a redirect response") + .into_response(); + assert_eq!(resp.status(), StatusCode::FOUND, "DASH request must be HTTP 302 Found redirect"); + let location = resp.headers().get(axum::http::header::LOCATION).expect("location header present"); + let location_str = location.to_str().expect("valid location string"); + assert!( + location_str.contains("provider-dash.example"), + "redirect location must point to provider URL, got {location_str}" + ); + } + + let dash_ttl = get_session_reservation_ttl_secs(&app_state, PlaylistItemType::LiveDash); + assert_eq!(dash_ttl, default_hls_session_ttl_secs(), "DASH preserves provider affinity TTL"); +} + +#[tokio::test] +async fn http_provider_lease_load_test_negative_control_detects_payload_corruption() { + const TS_PACKET_VALID: [u8; 188] = [0x47; 188]; + const TS_PACKET_CORRUPT: [u8; 188] = [0; 188]; + + let (origin_addr, origin_task) = + spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(TS_PACKET_CORRUPT.to_vec()), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_neg_ctrl".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + let addr = SocketAddr::from(([127, 0, 0, 1], 55_650)); + let fingerprint = create_test_fingerprint(addr); + let session = load_test_session(&input, origin_addr, "neg-session", addr); + let channel = load_test_channel(&input, origin_addr); + let user = load_test_user("neg_ctrl_user"); + + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let status = response.status(); + let body = response.into_body().collect().await.expect("body collects").to_bytes(); + assert!( + validate_load_response(status, &body, &TS_PACKET_VALID).is_err(), + "negative control must be rejected by the same validator as the load harness" + ); + + let _ = origin_task.await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn http_provider_lease_smoke_test() { + const TS_PACKET: [u8; 188] = [0x47; 188]; + let origin_addr = spawn_load_test_origin(&TS_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_smoke".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + for concurrency in [1usize, 2] { + let rounds = 5; + let mut tasks = tokio::task::JoinSet::new(); + for task_idx in 0..concurrency { + let app = Arc::clone(&app); + let input = Arc::clone(&input); + let user = load_test_user(&format!("smoke-user-{task_idx}")); + tasks.spawn(async move { + for r in 0..rounds { + let addr = SocketAddr::from(([127, 0, 0, 1], 48_000 + (task_idx as u16))); + let fingerprint = create_test_fingerprint(addr); + let token = format!("smoke-{task_idx}-{r}"); + let session = load_test_session(&input, origin_addr, &token, addr); + let channel = load_test_channel(&input, origin_addr); + + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let status = response.status(); + let body = response.into_body().collect().await.expect("smoke body collects").to_bytes(); + validate_load_response(status, &body, &TS_PACKET).expect("smoke response must match the TS packet"); + } + }); + } + while let Some(res) = tasks.join_next().await { + res.expect("smoke task must succeed"); + } + } + + for _ in 0..50 { + if app.active_provider.get_provider_connections_count() == 0 + && app.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "smoke slots baseline"); + assert_eq!(app.active_provider.provider_lease_usage(&input.name).total(), 0, "smoke lease baseline"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn http_provider_lease_downstream_smoke_test() { + const TS_PACKET: [u8; 188] = [0x47; 188]; + let origin_addr = spawn_load_test_origin(&TS_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_downstream".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + #[derive(Clone)] + struct DownstreamState { + app: Arc, + input: Arc, + origin_addr: SocketAddr, + } + + async fn stream_handler( + State(state): State, + ConnectInfo(addr): ConnectInfo, + ) -> axum::response::Response { + let fingerprint = create_test_fingerprint(addr); + let user = load_test_user("downstream-user"); + let session = load_test_session(&state.input, state.origin_addr, "downstream-token", addr); + let channel = load_test_channel(&state.input, state.origin_addr); + force_provider_stream_response( + &fingerprint, + &state.app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &state.input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response() + } + + let router = Router::new().route("/stream", get(stream_handler)).with_state(DownstreamState { + app: Arc::clone(&app), + input: Arc::clone(&input), + origin_addr, + }); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("downstream listener binds"); + let downstream_addr = listener.local_addr().expect("downstream address"); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, router.into_make_service_with_connect_info::()).await; + }); + + let client = reqwest::Client::new(); + for concurrency in [1usize, 2] { + let mut tasks = tokio::task::JoinSet::new(); + for task_idx in 0..concurrency { + let client = client.clone(); + let url = format!("http://{downstream_addr}/stream"); + tasks.spawn(async move { + for round in 0..3 { + let response = client.get(&url).send().await.expect("downstream response"); + let status = response.status(); + let body = response.bytes().await.expect("downstream body"); + validate_load_response(status, &body, &TS_PACKET) + .unwrap_or_else(|err| panic!("downstream task {task_idx} round {round}: {err}")); + } + }); + } + while let Some(res) = tasks.join_next().await { + res.expect("downstream task must succeed"); + } + } + + for _ in 0..50 { + if app.active_provider.get_provider_connections_count() == 0 + && app.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "downstream slots baseline"); + assert_eq!(app.active_provider.provider_lease_usage(&input.name).total(), 0, "downstream lease baseline"); + + server.abort(); +} + #[test] fn test_regular_response_keeps_compression_enabled() { let response = Response::new(()); @@ -1058,16 +3814,13 @@ async fn resolve_streaming_strategy_honors_forced_provider_fallback_policy() { let fallback_addr: SocketAddr = "127.0.0.1:55303".parse().unwrap_or_else(|_| unreachable!()); let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; - let busy = app_state - .active_provider - .acquire_exact_connection_with_grace( - &pinned_provider, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .await; + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &pinned_provider, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); assert!(busy.is_some(), "setup should occupy the pinned provider"); let strict = resolve_streaming_strategy( @@ -1082,6 +3835,7 @@ async fn resolve_streaming_strategy_honors_forced_provider_fallback_policy() { user_priority: 0, connection_kind: crate::api::model::ConnectionKind::Normal, session_owner: Some("vod-session"), + playback_kind: crate::model::PlaybackKind::Vod, accept_requested_stream_url: false, }, ) @@ -1107,6 +3861,7 @@ async fn resolve_streaming_strategy_honors_forced_provider_fallback_policy() { user_priority: 0, connection_kind: crate::api::model::ConnectionKind::Normal, session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, accept_requested_stream_url: false, }, ) @@ -1118,9 +3873,9 @@ async fn resolve_streaming_strategy_honors_forced_provider_fallback_policy() { }; assert_eq!(fallback_provider.as_ref(), "provider_2"); - app_state.active_provider.release_connection(&busy_addr).await; - app_state.active_provider.release_connection(&strict_addr).await; - app_state.active_provider.release_connection(&fallback_addr).await; + app_state.active_provider.release_connection(&busy_addr); + app_state.active_provider.release_connection(&strict_addr); + app_state.active_provider.release_connection(&fallback_addr); } #[tokio::test] @@ -1143,6 +3898,7 @@ async fn resolve_streaming_strategy_rewrites_stale_alias_url_to_selected_main_pr user_priority: 0, connection_kind: crate::api::model::ConnectionKind::Normal, session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, accept_requested_stream_url: false, }, ) @@ -1154,7 +3910,7 @@ async fn resolve_streaming_strategy_rewrites_stale_alias_url_to_selected_main_pr assert_eq!(provider.as_ref(), "provider_1"); assert_eq!(url.as_ref(), "http://provider-1.example/live/user1/pass1/100.ts"); - app_state.active_provider.release_connection(&addr).await; + app_state.active_provider.release_connection(&addr); } #[tokio::test] @@ -1187,16 +3943,13 @@ async fn resolve_streaming_strategy_rewrites_opaque_m3u_token_after_alias_alloca let busy_addr: SocketAddr = "127.0.0.1:55306".parse().unwrap_or_else(|_| unreachable!()); let alias_addr: SocketAddr = "127.0.0.1:55307".parse().unwrap_or_else(|_| unreachable!()); - let busy = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_a, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .await; + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &provider_a, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); assert!(busy.is_some(), "setup should occupy provider A"); let strategy = resolve_streaming_strategy( @@ -1211,6 +3964,7 @@ async fn resolve_streaming_strategy_rewrites_opaque_m3u_token_after_alias_alloca user_priority: 0, connection_kind: crate::api::model::ConnectionKind::Normal, session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, accept_requested_stream_url: false, }, ) @@ -1222,8 +3976,8 @@ async fn resolve_streaming_strategy_rewrites_opaque_m3u_token_after_alias_alloca assert_eq!(provider.as_ref(), "provider-b"); assert_eq!(url.as_ref(), "http://stream.example/channel/segment.ts?token=provider-b-token"); - app_state.active_provider.release_connection(&busy_addr).await; - app_state.active_provider.release_connection(&alias_addr).await; + app_state.active_provider.release_connection(&busy_addr); + app_state.active_provider.release_connection(&alias_addr); } #[tokio::test] @@ -1246,6 +4000,7 @@ async fn resolve_streaming_strategy_rejects_unmapped_provider_url() { user_priority: 0, connection_kind: crate::api::model::ConnectionKind::Normal, session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, accept_requested_stream_url: false, }, ) @@ -1257,7 +4012,7 @@ async fn resolve_streaming_strategy_rejects_unmapped_provider_url() { ProviderStreamState::Custom { reason: ProviderStreamCustomReason::UnmappedProviderUrl, .. } )); - app_state.active_provider.release_connection(&addr).await; + app_state.active_provider.release_connection(&addr); } #[tokio::test] @@ -1290,6 +4045,7 @@ async fn resolve_streaming_strategy_accepts_stalker_portal_url() { user_priority: 0, connection_kind: crate::api::model::ConnectionKind::Normal, session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, accept_requested_stream_url: false, }, ) @@ -1299,7 +4055,7 @@ async fn resolve_streaming_strategy_accepts_stalker_portal_url() { assert_eq!(provider.as_ref(), "provider_1"); assert_eq!(url.as_ref(), stream_url); - app_state.active_provider.release_connection(&addr).await; + app_state.active_provider.release_connection(&addr); } #[tokio::test] @@ -1322,6 +4078,7 @@ async fn resolve_streaming_strategy_accepts_session_requested_stream_url() { user_priority: 0, connection_kind: crate::api::model::ConnectionKind::Normal, session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, accept_requested_stream_url: true, }, ) @@ -1333,7 +4090,7 @@ async fn resolve_streaming_strategy_accepts_session_requested_stream_url() { assert_eq!(provider.as_ref(), "provider_1"); assert_eq!(url.as_ref(), trusted_url); - app_state.active_provider.release_connection(&addr).await; + app_state.active_provider.release_connection(&addr); } #[test] @@ -1525,7 +4282,7 @@ fn create_test_app_state_for_config(app_cfg: Arc) -> Arc { let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); let history_config = Some(StreamHistoryConfig::default()); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let config = app_cfg.config.load(); @@ -1898,6 +4655,8 @@ async fn activate_session_before_stream_open_skips_placeholder_for_follow_up_ses throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), }); let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!()); @@ -1961,8 +4720,8 @@ async fn activate_session_before_stream_open_skips_placeholder_for_follow_up_ses ) .await; - assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); - assert_eq!(activation.admission.kind, Some(crate::api::model::ConnectionKind::Normal)); + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(activation.admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); assert_eq!(activation.grace_mode, None); assert!( activation.placeholder_transition_version.is_none(), @@ -1985,6 +4744,8 @@ async fn activate_session_before_stream_open_revalidates_precomputed_follow_up_r throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), }); let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!()); @@ -2030,8 +4791,8 @@ async fn activate_session_before_stream_open_revalidates_precomputed_follow_up_r ) .await; - assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); - assert_eq!(activation.admission.kind, Some(crate::api::model::ConnectionKind::Normal)); + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(activation.admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); assert_eq!(activation.grace_mode, None); assert!( activation.placeholder_transition_version.is_some(), @@ -2057,6 +4818,8 @@ async fn activate_session_before_stream_open_stale_follow_up_reclassified_on_cou throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), }); let addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); @@ -2132,6 +4895,8 @@ async fn activate_session_before_stream_open_pre_resolved_grace_period_materiali throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }); let addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); @@ -2179,7 +4944,7 @@ async fn activate_session_before_stream_open_pre_resolved_grace_period_materiali ) .await; - assert_eq!(activation.admission.permission, UserConnectionPermission::GracePeriod); + assert_eq!(activation.admission.permission(), UserConnectionPermission::GracePeriod); assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); let session = app_state.active_users.get_and_update_user_session(&user.username, "tok-pre-resolved-grace").await; @@ -2205,6 +4970,8 @@ async fn activate_session_before_stream_open_skips_placeholder_for_prepare() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), }); let addr: SocketAddr = "127.0.0.1:55222".parse().unwrap_or_else(|_| unreachable!()); @@ -2234,7 +5001,7 @@ async fn activate_session_before_stream_open_skips_placeholder_for_prepare() { .await; // Prepare returns Allowed without running admission strategies. - assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); assert_eq!(activation.grace_mode, None); assert!( activation.placeholder_transition_version.is_none(), @@ -2258,6 +5025,8 @@ async fn resolve_playback_request_admission_prepare_only_returns_prepare_class() throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }); let addr: SocketAddr = "127.0.0.1:55223".parse().unwrap_or_else(|_| unreachable!()); @@ -2281,7 +5050,7 @@ async fn resolve_playback_request_admission_prepare_only_returns_prepare_class() assert_eq!(request_class, PlaybackRequestClass::Prepare); // Prepare returns Allowed without running strategies. - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(grace_mode, None); } @@ -2302,6 +5071,8 @@ async fn resolve_playback_request_admission_terminate_returns_terminate_class() throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }); let addr: SocketAddr = "127.0.0.1:55224".parse().unwrap_or_else(|_| unreachable!()); @@ -2345,7 +5116,7 @@ async fn resolve_playback_request_admission_terminate_returns_terminate_class() .await; assert_eq!(request_class, PlaybackRequestClass::Terminate); - assert_eq!(admission.permission, UserConnectionPermission::Exhausted); + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); assert_eq!(grace_mode, None); // Session should be expired after terminate. @@ -2379,6 +5150,8 @@ async fn activate_session_before_stream_open_marks_pending_provider_for_grace_ho throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }; let mut app_cfg = create_test_app_config(); @@ -2450,7 +5223,7 @@ async fn activate_session_before_stream_open_marks_pending_provider_for_grace_ho ) .await; - assert_eq!(activation.admission.permission, UserConnectionPermission::GracePeriod); + assert_eq!(activation.admission.permission(), UserConnectionPermission::GracePeriod); assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); let session = app_state @@ -2485,6 +5258,8 @@ async fn activate_session_before_stream_open_does_not_commit_user_lease_before_p throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: None, }; let mut app_cfg = create_test_app_config(); @@ -2532,7 +5307,7 @@ async fn activate_session_before_stream_open_does_not_commit_user_lease_before_p ) .await; - assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); assert_eq!(activation.grace_mode, None); let session = app_state @@ -2607,6 +5382,8 @@ async fn effective_admission_strategies_use_legacy_grace_when_field_missing() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: None, }); @@ -2631,6 +5408,8 @@ async fn effective_admission_strategies_respect_explicit_empty_list() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![]), }); @@ -2654,6 +5433,8 @@ async fn grace_context_is_populated_when_grace_strategy_is_actually_granted() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![ AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::GraceHoldStream, @@ -2729,7 +5510,7 @@ async fn grace_context_is_populated_when_grace_strategy_is_actually_granted() { ) .await; - assert_eq!(result.admission.permission, UserConnectionPermission::GracePeriod, "grace should be granted"); + assert_eq!(result.admission.permission(), UserConnectionPermission::GracePeriod, "grace should be granted"); assert!(matches!(result.grace_mode, Some(crate::api::model::GraceMode::Hold))); let ctx = result.grace_context.expect("grace_context must be present when grace is granted"); assert_eq!(ctx.strategy_index, 1, "GraceHoldStream is at index 1"); @@ -2758,6 +5539,8 @@ async fn evaluate_remaining_strategies_evicts_after_used_grace() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), }); @@ -2826,7 +5609,11 @@ async fn evaluate_remaining_strategies_evicts_after_used_grace() { ) .await; - assert_eq!(result.admission.permission, UserConnectionPermission::Allowed, "EvictUserOldest should free the slot"); + assert_eq!( + result.admission.permission(), + UserConnectionPermission::Allowed, + "EvictUserOldest should free the slot" + ); assert!(result.grace_context.is_none(), "no grace context on eviction success"); } @@ -2856,6 +5643,8 @@ async fn evaluate_remaining_strategies_skips_no_match_and_uses_later_eviction() throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![ AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserSameIpOldest, @@ -2929,7 +5718,7 @@ async fn evaluate_remaining_strategies_skips_no_match_and_uses_later_eviction() .await; assert_eq!( - result.admission.permission, + result.admission.permission(), UserConnectionPermission::Allowed, "EvictUserSameIpOldest should NoMatch, EvictUserOldest should succeed" ); @@ -2955,6 +5744,8 @@ async fn evaluate_remaining_strategies_empty_slice_denies() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }); @@ -2979,7 +5770,7 @@ async fn evaluate_remaining_strategies_empty_slice_denies() { ) .await; - assert_eq!(result.admission.permission, UserConnectionPermission::Exhausted, "empty remaining slice should deny"); + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted, "empty remaining slice should deny"); } #[tokio::test] @@ -3007,6 +5798,8 @@ async fn evaluate_remaining_strategies_preserves_soft_kind_on_exhausted() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }); @@ -3031,9 +5824,9 @@ async fn evaluate_remaining_strategies_preserves_soft_kind_on_exhausted() { ) .await; - assert_eq!(result.admission.permission, UserConnectionPermission::Exhausted, "empty remaining slice should deny"); + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted, "empty remaining slice should deny"); assert_eq!( - result.admission.kind, + result.admission.kind(), Some(crate::api::model::ConnectionKind::Soft), "exhausted result must preserve the original Soft connection kind" ); @@ -3066,6 +5859,8 @@ async fn evaluate_remaining_strategies_does_not_retry_used_prefix() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(strategies_for_config), }); @@ -3135,7 +5930,7 @@ async fn evaluate_remaining_strategies_does_not_retry_used_prefix() { .await; assert_eq!( - result.admission.permission, + result.admission.permission(), UserConnectionPermission::Allowed, "only EvictUserOldest should be evaluated, not GraceHoldStream" ); @@ -3167,6 +5962,8 @@ async fn evaluate_remaining_strategies_empty_slice_uses_original_kind_not_contex throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }); @@ -3191,9 +5988,10 @@ async fn evaluate_remaining_strategies_empty_slice_uses_original_kind_not_contex ) .await; - assert_eq!(result.admission.permission, UserConnectionPermission::Exhausted); + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); assert_eq!( - result.admission.kind, original_kind, + result.admission.kind(), + original_kind, "exhausted result must use original_kind (Soft), not grace_context.kind (Normal)" ); } @@ -3228,6 +6026,8 @@ async fn evaluate_remaining_strategies_later_grace_uses_original_kind_not_contex throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::GraceInstantStream]), }); @@ -3297,7 +6097,7 @@ async fn evaluate_remaining_strategies_later_grace_uses_original_kind_not_contex .await; assert_eq!( - result.admission.permission, + result.admission.permission(), UserConnectionPermission::GracePeriod, "remaining GraceInstantStream should grant GracePeriod" ); @@ -3325,6 +6125,8 @@ async fn resolve_admission_with_strategies_falls_through_after_failed_grace_gran throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), }); @@ -3427,8 +6229,8 @@ async fn resolve_admission_with_strategies_falls_through_after_failed_grace_gran let admission = result.admission; let grace_mode = result.grace_mode; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); - assert_eq!(admission.kind, Some(crate::api::model::ConnectionKind::Normal)); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); assert_eq!(grace_mode, None); } @@ -3489,7 +6291,7 @@ async fn resolve_admission_with_strategies_allows_existing_session_even_when_use }, ) .await; - assert_eq!(session_based.admission.permission, UserConnectionPermission::Allowed); + assert_eq!(session_based.admission.permission(), UserConnectionPermission::Allowed); let connection_based = resolve_admission_with_strategies( &app_state.admission_ctx(), @@ -3506,7 +6308,7 @@ async fn resolve_admission_with_strategies_allows_existing_session_even_when_use }, ) .await; - assert_eq!(connection_based.admission.permission, UserConnectionPermission::Exhausted); + assert_eq!(connection_based.admission.permission(), UserConnectionPermission::Exhausted); } #[tokio::test] @@ -3524,6 +6326,8 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), }); @@ -3538,9 +6342,41 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi let mut winner_channel = create_test_live_channel("http://provider-1.example/live/9002.ts"); winner_channel.virtual_id = 9002; + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "loop-user".to_string(); + app_state.connection_manager.add_connection(&victim_addr).await; app_state.connection_manager.add_connection(&winner_addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9001, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9001.ts", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-winner", + virtual_id: 9002, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9002.ts", + addr: &winner_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state .connection_manager .update_connection(crate::api::model::ConnectionParams { @@ -3578,7 +6414,7 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi app_state .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, RECENT_EVICTION_REENTRY_TTL_SECS) + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) .await; app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; @@ -3600,11 +6436,60 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi let admission = result.admission; let grace_mode = result.grace_mode; - assert_eq!(admission.permission, UserConnectionPermission::Exhausted); + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(admission.rejection_reason(), Some(AdmissionRejectionReason::RecentEvictionReentry)); + assert!(admission.is_reentry_suppressed()); assert_eq!(grace_mode, None); let active_streams = app_state.active_users.active_streams().await; assert_eq!(active_streams.len(), 1); assert_eq!(active_streams[0].channel.virtual_id, 9002); + + // A player may retry the winning plain-TS stream on another socket before the + // previous HTTP body has drained. That is a replacement of the same playback, + // not the evicted channel trying to reclaim its slot. + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&winner_addr, reconnect_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&winner_addr).await; + app_state.connection_manager.add_connection(&reconnect_addr).await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 3, + username: "loop-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &reconnect_fingerprint, + provider: "provider-a".intern(), + stream_channel: &winner_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }) + .await; + + let retry_addr: std::net::SocketAddr = "127.0.0.1:55184".parse().unwrap_or_else(|_| unreachable!()); + let retry = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "loop-user", + max_connections: 1, + soft_connections: 0, + client_ip: &reconnect_fingerprint.client_ip, + request_addr: &retry_addr, + use_session_admission: true, + session_token: Some("socket-retry"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9002) }, + }, + ) + .await; + + assert_eq!(retry.admission.permission(), UserConnectionPermission::Allowed); + assert!(app_state.active_users.active_streams().await.is_empty()); } #[tokio::test] @@ -3623,6 +6508,8 @@ async fn resolve_admission_with_strategies_allows_other_channel_after_recent_evi throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), }); @@ -3709,7 +6596,7 @@ async fn resolve_admission_with_strategies_allows_other_channel_after_recent_evi app_state .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, RECENT_EVICTION_REENTRY_TTL_SECS) + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) .await; app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; @@ -3730,7 +6617,7 @@ async fn resolve_admission_with_strategies_allows_other_channel_after_recent_evi .await; let admission = result.admission; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert!(app_state.active_users.active_streams().await.is_empty()); } @@ -3750,6 +6637,8 @@ async fn resolve_admission_with_strategies_does_not_suppress_different_session_o throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), }); @@ -3835,7 +6724,7 @@ async fn resolve_admission_with_strategies_does_not_suppress_different_session_o app_state .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, RECENT_EVICTION_REENTRY_TTL_SECS) + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) .await; app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; @@ -3857,7 +6746,7 @@ async fn resolve_admission_with_strategies_does_not_suppress_different_session_o let admission = result.admission; let grace_mode = result.grace_mode; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(grace_mode, None); assert!(app_state.active_users.active_streams().await.is_empty()); } @@ -3877,6 +6766,8 @@ async fn resolve_admission_with_strategies_allows_recently_evicted_playback_when throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), }); @@ -3931,7 +6822,7 @@ async fn resolve_admission_with_strategies_allows_recently_evicted_playback_when app_state .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, RECENT_EVICTION_REENTRY_TTL_SECS) + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) .await; app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; @@ -3953,8 +6844,8 @@ async fn resolve_admission_with_strategies_allows_recently_evicted_playback_when let admission = result.admission; let grace_mode = result.grace_mode; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); - assert_eq!(admission.kind, Some(crate::api::model::ConnectionKind::Soft)); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); assert_eq!(grace_mode, None); let active_streams = app_state.active_users.active_streams().await; @@ -4099,7 +6990,7 @@ async fn local_stream_response_rechecks_limits_before_registering_socket_bound_s &second_token ) .await - .permission, + .permission(), UserConnectionPermission::Exhausted, "failed second open must not leave a placeholder session that bypasses admission" ); @@ -4117,8 +7008,13 @@ async fn stream_response_preserves_soft_kind_for_shared_reuse() { let owner_handle = app_state .active_provider .acquire_connection(&input.name, &owner_addr, 0, crate::api::model::ConnectionKind::Normal) - .await .expect("owner allocation should exist"); + let subscriber_id = + tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); + let pending_cleanup = + SharedStreamManager::reserve_subscriber_cleanup(&app_state.connection_manager, subscriber_id, owner_addr) + .await + .expect("shared cleanup admission should succeed"); let shared_stream = stream::pending::>(); let registered = SharedStreamManager::register_shared_stream( SharedStreamCtx { @@ -4130,9 +7026,11 @@ async fn stream_response_preserves_soft_kind_for_shared_reuse() { stream_url, shared_stream, &owner_addr, + subscriber_id, Vec::new(), 1, - Some(owner_handle), + Some(tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), owner_handle)), + pending_cleanup, 0, crate::api::model::ConnectionKind::Normal, ) @@ -4172,8 +7070,8 @@ async fn stream_response_preserves_soft_kind_for_shared_reuse() { let admission = app_state.active_users.connection_admission(&user.username, user.max_connections, user.soft_connections).await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); - assert_eq!(admission.kind, Some(crate::api::model::ConnectionKind::Soft)); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); let soft_addr = "127.0.0.1:55142".parse().unwrap_or_else(|_| unreachable!()); let soft_fingerprint = create_test_fingerprint(soft_addr); @@ -4189,8 +7087,8 @@ async fn stream_response_preserves_soft_kind_for_shared_reuse() { &input, &target, &user, - admission.permission, - admission.kind.unwrap_or(crate::api::model::ConnectionKind::Normal), + admission.permission(), + admission.kind().unwrap_or(crate::api::model::ConnectionKind::Normal), false, None, ) @@ -4202,7 +7100,140 @@ async fn stream_response_preserves_soft_kind_for_shared_reuse() { .active_users .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, "soft-session") .await; - assert_eq!(session_admission.kind, Some(crate::api::model::ConnectionKind::Soft)); + assert_eq!(session_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); +} + +#[tokio::test] +async fn failed_provider_open_preserves_other_allocation_on_same_socket() -> Result<(), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let upstream = listener.local_addr()?; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + socket.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await + }); + let mut config = create_test_provider_app_config(); + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + enabled: true, + input_type: InputType::Xtream, + url: format!("http://{upstream}"), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + max_connections: 2, + ..ConfigInput::default() + }); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + let addr = "127.0.0.1:55144".parse()?; + let live = app + .active_provider + .acquire_connection(&input.name, &addr, 0, crate::api::model::ConnectionKind::Normal) + .ok_or("live allocation missing")?; + let url = format!("http://{upstream}/live/user1/pass1/100.ts"); + let channel = create_test_live_channel(&url); + let details = create_stream_response_details( + &app, + &get_stream_options(&app.app_config), + &url, + "failing-user", + &create_test_fingerprint(addr), + &HeaderMap::new(), + &input, + &channel, + PlaylistItemType::Live, + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + false, + UserConnectionPermission::Allowed, + None, + true, + false, + VirtualId::new(channel.virtual_id), + 0, + crate::api::model::ConnectionKind::Normal, + false, + Some("failed-session"), + None, + false, + None, + None, + ) + .await?; + assert!(details.provider_handle.is_none()); + assert_eq!(app.active_provider.get_provider_connections_count(), 1); + assert!(!live.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + app.active_provider.release_handle(&live); + server.await??; + Ok(()) +} + +#[tokio::test] +async fn ts_direct_capacity_spreads_across_priority_groups_then_rejects_sixth() { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_a".intern(), + enabled: true, + input_type: InputType::Xtream, + url: "http://127.0.0.1:1".to_string(), + username: Some("user-a".to_string()), + password: Some("pass-a".to_string()), + priority: 0, + max_connections: 2, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_b".intern(), + url: "http://127.0.0.1:2".to_string(), + username: Some("user-b".to_string()), + password: Some("pass-b".to_string()), + priority: 1, + max_connections: 3, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + let addr: SocketAddr = "127.0.0.1:55120".parse().unwrap_or_else(|_| unreachable!()); + let mut handles = Vec::new(); + for _ in 0..5 { + let handle = app.active_provider.acquire_connection_with_grace( + &input.name, + &addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(handle.is_some(), "five allocations must succeed against A(2)+B(3)"); + handles.push(handle.expect("allocation present")); + } + + let sixth = app.active_provider.acquire_connection_with_grace( + &input.name, + &addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(sixth.is_none(), "sixth start must be rejected once A(2)+B(3) are exhausted"); + + let active = app.active_provider.active_connections().expect("active connection map present"); + assert_eq!(active.get("provider_a").copied(), Some(5), "multi lineup must hold five allocations in total"); + + let capacities = app.active_provider.provider_capacities_for_input(&input.name); + let current_of = |name: &str| capacities.iter().find(|(n, _, _)| n.as_ref() == name).map(|(_, cur, _)| *cur); + assert_eq!(current_of("provider_a"), Some(2), "priority group A must hold two allocations"); + assert_eq!(current_of("provider_b"), Some(3), "priority group B must hold three allocations"); + + for handle in &handles { + app.active_provider.release_handle(handle); + } + assert_eq!(app.active_provider.get_provider_connections_count(), 0); } #[tokio::test] @@ -4289,6 +7320,7 @@ async fn stream_response_rolls_back_provisional_user_activation_when_provider_op async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_video() { let app_state = create_test_app_state(); let no_video_details = StreamDetails { + shared_subscriber_id: None, stream: None, stream_info: Some((Vec::new(), StatusCode::OK, None, None)), provider_name: Some("provider_1".intern()), @@ -4303,6 +7335,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; assert!( should_pin_provider_for_session(&no_video_details, &app_state, PlaylistItemType::Catchup), @@ -4310,6 +7343,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid ); let provisioning_details = StreamDetails { + shared_subscriber_id: None, stream: None, stream_info: Some((Vec::new(), StatusCode::OK, None, Some(CustomVideoStreamType::Provisioning))), provider_name: Some("provider_1".intern()), @@ -4324,6 +7358,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; assert!( should_pin_provider_for_session(&provisioning_details, &app_state, PlaylistItemType::Catchup), @@ -4338,6 +7373,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid CustomVideoStreamType::LowPriorityPreempted, ] { let failure_details = StreamDetails { + shared_subscriber_id: None, stream: None, stream_info: Some((Vec::new(), StatusCode::BAD_REQUEST, None, Some(failure_type))), provider_name: Some("provider_1".intern()), @@ -4352,6 +7388,7 @@ async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_vid provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; assert!( !should_pin_provider_for_session(&failure_details, &app_state, PlaylistItemType::Catchup), @@ -4638,7 +7675,7 @@ async fn local_stream_response_preserves_soft_kind_across_reopens() { playback_session_token, ) .await; - assert_eq!(session_admission.kind, Some(crate::api::model::ConnectionKind::Soft)); + assert_eq!(session_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); } #[tokio::test] @@ -4713,8 +7750,8 @@ async fn activated_session_admission_keeps_hls_placeholders_uncounted_via_api_ut ) .await; - assert_eq!(first_admission.admission.permission, UserConnectionPermission::Allowed); - assert_eq!(second_admission.admission.permission, UserConnectionPermission::Allowed); + assert_eq!(first_admission.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(second_admission.admission.permission(), UserConnectionPermission::Allowed); assert_eq!(app_state.active_users.user_connections(&user.username).await, 0); } @@ -4773,7 +7810,7 @@ async fn socket_bound_playback_sessions_enforce_hard_limits_per_socket() { .active_users .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &second_token) .await; - assert_eq!(admission.permission, UserConnectionPermission::Exhausted); + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); } #[tokio::test] @@ -4792,6 +7829,8 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![ AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserSameIpLatest, @@ -4869,7 +7908,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same let admission = result.admission; let grace_mode = result.grace_mode; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(grace_mode, None); assert!(app_state.active_users.active_streams().await.is_empty()); assert_eq!( @@ -4945,8 +7984,8 @@ async fn socket_bound_playback_sessions_still_allow_soft_slots() { .active_users .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &second_token) .await; - assert_eq!(second_admission.permission, UserConnectionPermission::Allowed); - assert_eq!(second_admission.kind, Some(crate::api::model::ConnectionKind::Soft)); + assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(second_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); app_state .active_users @@ -4985,7 +8024,7 @@ async fn socket_bound_playback_sessions_still_allow_soft_slots() { .active_users .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &third_token) .await; - assert_eq!(third_admission.permission, UserConnectionPermission::Exhausted); + assert_eq!(third_admission.permission(), UserConnectionPermission::Exhausted); } #[test] @@ -5045,6 +8084,21 @@ fn playback_session_fingerprint_keeps_ts_socket_bound_but_vod_logical() { assert_eq!(first_vod, second_vod, "VOD remains logical across reopen/seek sockets"); } +#[test] +fn catchup_session_key_is_sticky_only_within_the_same_archive_window() { + let addr: SocketAddr = "127.0.0.1:55181".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = Fingerprint::new("10.0.0.8|player".to_string(), "10.0.0.8".to_string(), addr); + + let first = create_catchup_session_key(&fingerprint, "user1", 7004, "/timeshift/3600/1700000000/"); + let same = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/3600/1700000000"); + let seeked = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/3600/1700000300"); + let resized = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/1800/1700000000"); + + assert_eq!(first, same, "cosmetic path separators must not break provider stickiness"); + assert_ne!(first, seeked, "seeking must identify the new archive window"); + assert_ne!(first, resized, "changing duration must identify the new archive window"); +} + #[tokio::test] #[allow(clippy::too_many_lines)] async fn xtream_hls_then_ts_uses_distinct_tokens_and_evicts_old_hls_session() { @@ -5069,6 +8123,8 @@ async fn xtream_hls_then_ts_uses_distinct_tokens_and_evicts_old_hls_session() { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![ AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserSameIpLatest, @@ -5156,7 +8212,7 @@ async fn xtream_hls_then_ts_uses_distinct_tokens_and_evicts_old_hls_session() { .active_users .connection_admission(&user.username, user.max_connections, user.soft_connections) .await - .permission, + .permission(), UserConnectionPermission::Exhausted, "the preserved HLS playback must still reserve the user's only slot before the TS request is evaluated" ); @@ -5179,7 +8235,7 @@ async fn xtream_hls_then_ts_uses_distinct_tokens_and_evicts_old_hls_session() { ) .await; assert_eq!(request_class, PlaybackRequestClass::Activate); - assert_eq!(ts_admission.permission, UserConnectionPermission::Allowed); + assert_eq!(ts_admission.permission(), UserConnectionPermission::Allowed); assert_eq!(ts_grace_mode, None); assert!( app_state @@ -5206,8 +8262,8 @@ async fn xtream_hls_then_ts_uses_distinct_tokens_and_evicts_old_hls_session() { provider: "provider_1", stream_url: ts_channel.url.as_ref(), addr: &ts_addr, - connection_permission: ts_admission.permission, - connection_kind: ts_admission.kind, + connection_permission: ts_admission.permission(), + connection_kind: ts_admission.kind(), socket_bound: true, }) .await; @@ -5300,7 +8356,6 @@ async fn intentional_deferred_open_retains_provider_grace_handle() { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("holder occupies the provider slot"); let input = app_state.app_config.get_input_by_name(&provider_name).expect("provider input"); let stream_url = "http://provider-1.example/live/user1/pass1/100.m3u8"; @@ -5341,8 +8396,8 @@ async fn intentional_deferred_open_retains_provider_grace_handle() { assert!(details.has_deferred_provider_open()); assert!(details.provider_handle.is_some(), "deferred open must retain its provider allocation"); - app_state.connection_manager.release_provider_handle(details.provider_handle.take()).await; - app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; + app_state.connection_manager.release_managed_provider_handle(details.provider_handle.take()); + app_state.connection_manager.release_provider_handle(Some(holder_handle)); } #[test] @@ -5361,7 +8416,7 @@ async fn forced_reopen_cleanup_for_adaptive_streams_does_not_close_client_socket let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!()); let mut close_rx = app_state.connection_manager.get_close_connection_channel(); - cleanup_forced_reopen_addrs(&app_state, PlaylistItemType::LiveHls, &[addr]).await; + cleanup_forced_reopen_addrs(&app_state, "adaptive-owner", &[addr]).await; let signal = tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); @@ -5369,19 +8424,16 @@ async fn forced_reopen_cleanup_for_adaptive_streams_does_not_close_client_socket } #[tokio::test] -async fn forced_reopen_cleanup_for_non_adaptive_streams_closes_client_socket() { +async fn forced_reopen_cleanup_for_non_adaptive_streams_preserves_client_socket() { let app_state = create_test_app_state(); let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!()); let mut close_rx = app_state.connection_manager.get_close_connection_channel(); - cleanup_forced_reopen_addrs(&app_state, PlaylistItemType::Live, &[addr]).await; + cleanup_forced_reopen_addrs(&app_state, "live-owner", &[addr]).await; let signal = tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); - assert!(matches!( - signal, - Some(crate::api::model::CloseConnectionSignal::WithReason(signal_addr, _)) if signal_addr == addr - )); + assert!(signal.is_none(), "a reopen must not close unrelated requests on the same proxy socket"); } #[tokio::test] @@ -5781,3 +8833,733 @@ fn evaluate_network_access_respects_allow_policy() { NetworkAccessDecision::AllowedGeoIpUnavailable ); } + +#[tokio::test] +async fn test_channel_unavailable_fallback_does_not_register_body_owner_or_leak_slot() { + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let mut cfg = (**app_state.app_config.config.load()).clone(); + cfg.custom_stream_response_enabled = true; + app_state.app_config.config.store(std::sync::Arc::new(cfg)); + let addr: std::net::SocketAddr = "127.0.0.1:49080".parse().unwrap(); + + let custom_video = crate::model::CustomStreamResponse { + channel_unavailable: Some(crate::api::model::TransportStreamBuffer::new(b"channel-unavailable-bytes".to_vec())), + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }; + app_state.app_config.custom_stream_response.store(Some(std::sync::Arc::new(custom_video))); + + let handle = app_state + .active_provider + .acquire_connection(&provider_name, &addr, 0, tuliprox_session::ConnectionKind::Normal) + .expect("connection should be acquired"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + let (channel_unavail_stream, info) = + create_channel_unavailable_stream(&app_state.app_config, &[], axum::http::StatusCode::OK); + assert!(channel_unavail_stream.is_some()); + + let factory_response = tuliprox_session::ProviderStreamFactoryResponse { + stream: channel_unavail_stream.unwrap(), + info, + provider_session_headers: std::collections::HashMap::new(), + has_upstream_owner: false, + }; + + if factory_response.has_upstream_owner { + app_state.active_provider.register_body_owner(handle.allocation_id); + } + + app_state.active_provider.release_handle(&handle); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::()).unwrap_or(0), 0); +} + +#[tokio::test] +async fn test_provider_open_cancelled_during_header_wait_releases_slot_without_leak() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + + let accepted = std::sync::Arc::new(tokio::sync::Notify::new()); + let accepted_clone = std::sync::Arc::clone(&accepted); + + let server_task = tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + accepted_clone.notify_waiters(); + let mut buf = [0u8; 1024]; + loop { + use tokio::io::AsyncReadExt; + match socket.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + } + }); + + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let client_addr: std::net::SocketAddr = "127.0.0.1:49081".parse().unwrap(); + + let handle = app_state + .active_provider + .acquire_connection(&provider_name, &client_addr, 0, tuliprox_session::ConnectionKind::Normal) + .expect("connection should be acquired"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + let url = url::Url::parse(&format!("http://{local_addr}/stream.ts")).unwrap(); + let req_headers = axum::http::HeaderMap::new(); + let stream_options = get_stream_options(&app_state.app_config); + let mut options = + crate::api::model::ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { + addr: client_addr, + item_type: PlaylistItemType::Live, + share_stream: false, + stream_options: &stream_options, + stream_url: &url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: tuliprox_session::ProviderContentRepresentationMode::PreserveOrigin, + }); + options.set_provider_handle_tokens( + handle.cancel_token.clone(), + handle.completion_token.clone(), + Some(handle.close_reason.clone()), + ); + app_state.active_provider.mark_opening(handle.allocation_id); + + let ctx = app_state.provider_stream_ctx(); + let client = app_state.http_client.load().as_ref().clone(); + + let open_task = + tokio::spawn(async move { crate::api::model::create_provider_stream(&ctx, &client, options).await }); + + // Wait until TCP connection is accepted and waiting for response headers + accepted.notified().await; + + // Now cancel the handle while it is waiting for response headers + handle.cancel_token.as_ref().unwrap().cancel(); + + // The open task should finish quickly because it is cancelled + let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), open_task) + .await + .expect("open task should abort upon cancellation") + .expect("task join"); + assert!(outcome.is_none()); + + // Release the handle + app_state.active_provider.release_handle(&handle); + + // Yield to allow background completion tasks to process + tokio::task::yield_now().await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::()).unwrap_or(0), 0); + + let _ = server_task.await; +} + +#[tokio::test] +async fn test_reentry_suppression_response_behavior() { + let response = crate::api::api_utils::reentry_suppressed_response(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + assert!(!response.headers().contains_key("x-tuliprox-rejection")); + assert!(!response.headers().contains_key("content-type")); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn test_eviction_strategy_falls_back_when_candidate_protected_by_reentry() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: SocketAddr = "127.0.0.1:55301".parse().unwrap_or_else(|_| unreachable!()); + let winner_addr: SocketAddr = "127.0.0.1:55302".parse().unwrap_or_else(|_| unreachable!()); + let other_ip_addr: SocketAddr = "10.0.0.1:55303".parse().unwrap_or_else(|_| unreachable!()); + let retry_addr: SocketAddr = "127.0.0.1:55304".parse().unwrap_or_else(|_| unreachable!()); + + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); + let other_ip_fingerprint = create_test_fingerprint_with_user_agent(other_ip_addr, "other/1.0"); + let retry_fingerprint = create_test_fingerprint_with_user_agent(retry_addr, "player/1.0"); + + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9501.ts"); + victim_channel.virtual_id = 9501; + let winner_channel = create_test_live_channel("http://provider-1.example/live/9502.ts"); + let other_channel = create_test_live_channel("http://provider-1.example/live/9503.ts"); + + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "fallback-user".to_string(); + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&winner_addr).await; + app_state.connection_manager.add_connection(&other_ip_addr).await; + + // Create user sessions so streams are linked to counted sessions + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9501, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9501.ts", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-winner", + virtual_id: 9502, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9502.ts", + addr: &winner_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-other", + virtual_id: 9503, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9503.ts", + addr: &other_ip_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Stream 0: victim on same IP (127.0.0.1), channel 9501 + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 99, + username: "fallback-user", + max_connections: 3, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &victim_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + + // Stream 1: winner on same IP (127.0.0.1), channel 9502 + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "fallback-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &winner_fingerprint, + provider: "provider-a".intern(), + stream_channel: &winner_channel, + user_agent: std::borrow::Cow::Borrowed("winner/1.0"), + session_token: Some("session-winner"), + }) + .await; + + // Stream 2: stream on different IP (10.0.0.1), channel 9503 + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "fallback-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &other_ip_fingerprint, + provider: "provider-a".intern(), + stream_channel: &other_channel, + user_agent: std::borrow::Cow::Borrowed("other/1.0"), + session_token: Some("session-other"), + }) + .await; + + // Mark winner_addr as protected against reentry of victim_addr (channel 9501) + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + // Retry for channel 9501 arrives from same IP (127.0.0.1). + // EvictUserSameIpOldest will target winner_addr (same IP), but winner_addr is protected! + // It should be suppressed, filtered out, and Fall B will continue to EvictUserOldest. + // EvictUserOldest will then target other_ip_addr (which is NOT protected). + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "fallback-user", + max_connections: 2, + soft_connections: 0, + client_ip: &retry_fingerprint.client_ip, + request_addr: &retry_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-retry"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9501) }, + }, + ) + .await; + + // Request must be Admitted because the fallback strategy evicted the unprotected stream! + assert_eq!(result.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(result.admission.rejection_reason(), None); + + // Now test rejection when NO other candidate exists (only the protected stream). + // EvictUserSameIpOldest suppresses winner_addr; EvictUserOldest finds no candidates left. + let result_exhausted = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "fallback-user", + max_connections: 1, + soft_connections: 0, + client_ip: &retry_fingerprint.client_ip, + request_addr: &retry_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-retry-2"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9501) }, + }, + ) + .await; + + assert_eq!(result_exhausted.admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(result_exhausted.admission.rejection_reason(), Some(AdmissionRejectionReason::RecentEvictionReentry)); + assert!(result_exhausted.admission.is_reentry_suppressed()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn test_mixed_suppressed_and_legitimate_eviction_reports_exhaustion() { + // A protected candidate is skipped, then a legitimate (but uncounted) candidate is + // evicted and admission still fails. That residual failure is real exhaustion and + // must not be mislabelled as `RecentEvictionReentry`. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: SocketAddr = "127.0.0.1:55321".parse().unwrap_or_else(|_| unreachable!()); + let protected_addr: SocketAddr = "127.0.0.1:55322".parse().unwrap_or_else(|_| unreachable!()); + let orphan_addr: SocketAddr = "10.0.0.1:55323".parse().unwrap_or_else(|_| unreachable!()); + let request_addr: SocketAddr = "127.0.0.1:55324".parse().unwrap_or_else(|_| unreachable!()); + + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let protected_fingerprint = create_test_fingerprint_with_user_agent(protected_addr, "protected/1.0"); + let orphan_fingerprint = create_test_fingerprint_with_user_agent(orphan_addr, "orphan/1.0"); + let request_fingerprint = create_test_fingerprint_with_user_agent(request_addr, "player/1.0"); + + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9701.ts"); + victim_channel.virtual_id = 9701; + let mut protected_channel = create_test_live_channel("http://provider-1.example/live/9702.ts"); + protected_channel.virtual_id = 9702; + let mut orphan_channel = create_test_live_channel("http://provider-1.example/live/9703.ts"); + orphan_channel.virtual_id = 9703; + + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "mixed-user".to_string(); + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&protected_addr).await; + app_state.connection_manager.add_connection(&orphan_addr).await; + + // The guard source: a counted stream on the requested channel (9701). It supplies the + // socket reentry key and is released before the retry is resolved. + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9701, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9701.ts", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 91, + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &victim_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + + // The protected candidate: counted, same IP as the retry, different channel. + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-protected", + virtual_id: 9702, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9702.ts", + addr: &protected_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 92, + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &protected_fingerprint, + provider: "provider-a".intern(), + stream_channel: &protected_channel, + user_agent: std::borrow::Cow::Borrowed("protected/1.0"), + session_token: Some("session-protected"), + }) + .await; + + // The legitimate fallback candidate: an orphan stream (no session), so evicting it + // frees no counted connection and admission stays exhausted. + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 93, + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &orphan_fingerprint, + provider: "provider-a".intern(), + stream_channel: &orphan_channel, + user_agent: std::borrow::Cow::Borrowed("orphan/1.0"), + session_token: None, + }) + .await; + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, protected_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + client_ip: &request_fingerprint.client_ip, + request_addr: &request_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-retry"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9701) }, + }, + ) + .await; + + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(result.admission.rejection_reason(), Some(AdmissionRejectionReason::UserConnectionsExhausted)); + assert!(!result.admission.is_reentry_suppressed()); +} + +fn connection_denied_count(app_state: &Arc) -> u64 { + app_state + .active_users + .events() + .stats() + .emitted() + .into_iter() + .find_map(|(kind, count)| (kind == shared::model::EventKind::ConnectionDenied).then_some(count)) + .unwrap_or(0) +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn test_reentry_suppression_does_not_emit_connection_denied() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: SocketAddr = "127.0.0.1:55331".parse().unwrap_or_else(|_| unreachable!()); + let protected_addr: SocketAddr = "127.0.0.1:55332".parse().unwrap_or_else(|_| unreachable!()); + let request_addr: SocketAddr = "127.0.0.1:55333".parse().unwrap_or_else(|_| unreachable!()); + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let protected_fingerprint = create_test_fingerprint_with_user_agent(protected_addr, "protected/1.0"); + let request_fingerprint = create_test_fingerprint_with_user_agent(request_addr, "player/1.0"); + + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9711.ts"); + victim_channel.virtual_id = 9711; + let mut protected_channel = create_test_live_channel("http://provider-1.example/live/9712.ts"); + protected_channel.virtual_id = 9712; + + let mut user = ProxyUserCredentials::default(); + user.username = "event-reentry-user".to_string(); + user.max_connections = 1; + user.soft_connections = 0; + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&protected_addr).await; + + for (token, virtual_id, url, addr, fingerprint, channel, meter_uid) in [ + ( + "session-victim", + 9711, + "http://provider-1.example/live/9711.ts", + &victim_addr, + &victim_fingerprint, + &victim_channel, + 201_u32, + ), + ( + "session-protected", + 9712, + "http://provider-1.example/live/9712.ts", + &protected_addr, + &protected_fingerprint, + &protected_channel, + 202_u32, + ), + ] { + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: token, + virtual_id, + provider: "provider-a", + stream_url: url, + addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid, + username: "event-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint, + provider: "provider-a".intern(), + stream_channel: channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(token), + }) + .await; + } + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, protected_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let (admission, _, _) = crate::api::api_utils::resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &request_fingerprint, + None, + "session-event", + false, + EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9711) }, + false, + false, + ) + .await; + + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); + assert!(admission.is_reentry_suppressed()); + assert_eq!(connection_denied_count(&app_state), 0, "a suppressed reentry must not emit ConnectionDenied"); + assert_eq!( + app_state.active_users.reentry_suppressed_total(), + 1, + "a suppressed reentry must be counted once as a diagnostic" + ); +} + +#[tokio::test] +async fn test_real_exhaustion_emits_connection_denied() { + // Negative control: a genuine connection-limit denial must still emit the event. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![]), + }); + + let addr: SocketAddr = "127.0.0.1:55341".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint_with_user_agent(addr, "player/1.0"); + let mut user = ProxyUserCredentials::default(); + user.username = "event-limit-user".to_string(); + user.max_connections = 1; + user.soft_connections = 0; + + app_state.connection_manager.add_connection(&addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "session-limit", + virtual_id: 9721, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9721.ts", + addr: &addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + // A stream row is what makes the session count towards `max_connections`. + let channel = create_test_live_channel("http://provider-1.example/live/9721.ts"); + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 211, + username: "event-limit-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-limit"), + }) + .await; + + let (admission, _, _) = crate::api::api_utils::resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &fingerprint, + None, + "session-new", + false, + EvictionReentryGuard::Session("session-new"), + false, + false, + ) + .await; + + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(admission.rejection_reason(), Some(AdmissionRejectionReason::UserConnectionsExhausted)); + assert_eq!(connection_denied_count(&app_state), 1, "a real limit denial must emit ConnectionDenied"); + assert_eq!( + app_state.active_users.reentry_suppressed_total(), + 0, + "a real limit denial must not be counted as reentry suppression" + ); +} diff --git a/backend/app/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs index 6b74fd73f..bd521d06f 100644 --- a/backend/app/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -687,7 +687,7 @@ mod tests { let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let config = app_cfg.config.load(); @@ -750,6 +750,41 @@ mod tests { }) } + #[tokio::test] + async fn custom_video_response_preserves_other_provider_on_proxy_socket() -> Result<(), Box> + { + let app = create_test_app_state(); + let addr = test_fingerprint().addr; + let input = app.app_config.sources.load().inputs.first().cloned().ok_or("input missing")?; + let handle = app + .active_provider + .acquire_connection(&input.name, &addr, 0, tuliprox_session::ConnectionKind::Normal) + .ok_or("allocation missing")?; + let response = crate::api::model::create_custom_video_stream_response( + &app.provider_stream_ctx(), + &addr, + crate::api::model::CustomVideoStreamType::ChannelUnavailable, + ) + .into_response(); + assert_eq!(response.status(), StatusCode::OK); + // Confirmation is a queue barrier for any cleanup emitted by the response. + app.active_provider.refresh_provider_reservation(&input.name, "cleanup-barrier", 30); + app.connection_manager.send_cleanup(tuliprox_session::CleanupEvent::ConfirmPlaybackLease { + owner: Arc::from("cleanup-barrier"), + request_id: None, + }); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app.active_provider.provider_lease_usage(&input.name).active == 0 { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(app.active_provider.get_provider_connections_count(), 1); + app.active_provider.clear_provider_reservation("cleanup-barrier"); + app.active_provider.release_handle(&handle); + Ok(()) + } + #[tokio::test] async fn custom_video_stream_response_returns_ok_for_channel_unavailable() { let app_state = create_test_app_state(); diff --git a/backend/app/src/api/endpoints/download_api/mod.rs b/backend/app/src/api/endpoints/download_api/mod.rs index abfa7d1ae..d2a2ec990 100644 --- a/backend/app/src/api/endpoints/download_api/mod.rs +++ b/backend/app/src/api/endpoints/download_api/mod.rs @@ -1161,7 +1161,7 @@ pub(in crate::api) async fn ensure_download_worker_running( }; if let Some(input_name) = input_name { loop { - let capacities = active_provider.provider_capacities_for_input(&input_name).await; + let capacities = active_provider.provider_capacities_for_input(&input_name); if background_download_should_wait(priority, &capacities, &download_cfg) { if let Err(err) = broadcast_worker_mutation( &event_manager, @@ -1199,7 +1199,7 @@ pub(in crate::api) async fn ensure_download_worker_running( continue; } if let Some(handle) = - active_provider.acquire_connection_for_download(&input_name, priority).await + active_provider.acquire_connection_for_download(&input_name, priority) { break ProviderAcquireResult::Acquired(Some(handle)); } @@ -1266,12 +1266,12 @@ pub(in crate::api) async fn ensure_download_worker_running( broadcast_download_queue_update(&event_manager, &dq).await; } Ok(None) => { - connection_manager.release_provider_handle(handle).await; + connection_manager.release_provider_handle(handle); error!("Download worker active task changed after provider acquire"); break 'worker; } Err(err) => { - connection_manager.release_provider_handle(handle).await; + connection_manager.release_provider_handle(handle); error!("Download worker commit failed after provider acquire: {err}"); break 'worker; } @@ -1321,7 +1321,7 @@ pub(in crate::api) async fn ensure_download_worker_running( let execution_result = { let Some(download) = active_download_snapshot_for_worker(&dq.active, &worker_uuid).await else { - connection_manager.release_provider_handle(provider_handle).await; + connection_manager.release_provider_handle(provider_handle); break 'worker; }; match download.kind.clone() { @@ -1406,7 +1406,7 @@ pub(in crate::api) async fn ensure_download_worker_running( match execution_result { DownloadExecutionResult::Completed => { - connection_manager.release_provider_handle(provider_handle).await; + connection_manager.release_provider_handle(provider_handle); let measured_bytes = { let active = dq.active.read().await; match active.as_ref() { @@ -1455,7 +1455,7 @@ pub(in crate::api) async fn ensure_download_worker_running( } } DownloadExecutionResult::Paused => { - connection_manager.release_provider_handle(provider_handle).await; + connection_manager.release_provider_handle(provider_handle); if let Err(err) = broadcast_required_worker_mutation( &event_manager, &dq, @@ -1471,7 +1471,7 @@ pub(in crate::api) async fn ensure_download_worker_running( break; } DownloadExecutionResult::Cancelled => { - connection_manager.release_provider_handle(provider_handle).await; + connection_manager.release_provider_handle(provider_handle); if let Err(err) = broadcast_required_worker_mutation( &event_manager, &dq, @@ -1485,7 +1485,7 @@ pub(in crate::api) async fn ensure_download_worker_running( } } DownloadExecutionResult::Preempted => { - connection_manager.release_provider_handle(provider_handle).await; + connection_manager.release_provider_handle(provider_handle); let control = *control_signal.read().await; match control { DownloadControl::Restart => warn!( @@ -1524,7 +1524,7 @@ pub(in crate::api) async fn ensure_download_worker_running( } } DownloadExecutionResult::Retryable(_err) => { - connection_manager.release_provider_handle(provider_handle).await; + connection_manager.release_provider_handle(provider_handle); warn!("Retrying active download after transient failure"); let retry_commit = prepare_active_retry(&dq, &worker_uuid, &download_cfg).await; let retry_delay_secs = match retry_commit { @@ -1641,7 +1641,7 @@ pub(in crate::api) async fn ensure_download_worker_running( } } DownloadExecutionResult::Failed(err) => { - connection_manager.release_provider_handle(provider_handle).await; + connection_manager.release_provider_handle(provider_handle); warn!("Download failed permanently: {err}"); let committed = finish_active_and_promote(&dq, &worker_uuid, |fd| { fd.finished = true; @@ -1729,7 +1729,7 @@ pub(in crate::api) fn start_download_scheduler( let capacities = if let Some(capacities) = capacities_by_input.get(input_name) { capacities.clone() } else { - let capacities = bridge_active_provider.provider_capacities_for_input(input_name).await; + let capacities = bridge_active_provider.provider_capacities_for_input(input_name); capacities_by_input.insert(Arc::clone(input_name), capacities.clone()); capacities }; diff --git a/backend/app/src/api/endpoints/download_api/tests.rs b/backend/app/src/api/endpoints/download_api/tests.rs index 5150ae15c..c5da1a14c 100644 --- a/backend/app/src/api/endpoints/download_api/tests.rs +++ b/backend/app/src/api/endpoints/download_api/tests.rs @@ -554,7 +554,7 @@ fn create_test_app_state_with_downloads(downloads: Arc) -> Arc::default()); let config = app_cfg.config.load(); diff --git a/backend/app/src/api/endpoints/hls_api/catchup.rs b/backend/app/src/api/endpoints/hls_api/catchup.rs index 483962b9e..d5a4f3f92 100644 --- a/backend/app/src/api/endpoints/hls_api/catchup.rs +++ b/backend/app/src/api/endpoints/hls_api/catchup.rs @@ -189,7 +189,7 @@ pub(super) async fn admit_recovered_archive_stream( .await, )); } - if app_state.active_provider.is_over_limit(&session.provider).await { + if app_state.active_provider.is_over_limit(&session.provider) { return Err(Box::new( hls_admission_failure_manifest_response( app_state, @@ -215,8 +215,8 @@ pub(super) async fn admit_recovered_archive_stream( false, ) .await; - let connection_permission = connection_admission.permission; - let connection_kind = connection_admission.kind.or(session.connection_kind); + let connection_permission = connection_admission.permission(); + let connection_kind = connection_admission.kind().or(session.connection_kind); session.permission = connection_permission; if let Some(connection_kind) = connection_kind { session.connection_kind = Some(connection_kind); @@ -224,6 +224,9 @@ pub(super) async fn admit_recovered_archive_stream( if connection_permission == UserConnectionPermission::Exhausted || (connection_permission == UserConnectionPermission::GracePeriod && connection_kind.is_none()) { + if connection_admission.is_reentry_suppressed() { + return Err(Box::new(crate::api::api_utils::reentry_suppressed_response())); + } let provider = if session.provider.is_empty() { input.name.clone() } else { session.provider.clone() }; return Err(Box::new( hls_admission_failure_manifest_response( diff --git a/backend/app/src/api/endpoints/hls_api/manifest.rs b/backend/app/src/api/endpoints/hls_api/manifest.rs index 80db2035d..d7419b568 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest.rs @@ -644,31 +644,29 @@ pub(super) async fn try_reserve_hls_entry_origin_account_for_redirect( request_url: &str, user_session_token: &str, session_owner: &str, + playback_kind: PlaybackKind, reservation_ttl_secs: u64, connection_permission: UserConnectionPermission, connection_kind: crate::api::model::ConnectionKind, create_user_session: bool, ) -> Option { - let provider_handle = app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &fingerprint.addr, - false, - connection_priority_for_kind(user, connection_kind), - connection_kind, - Some(session_owner), - ) - .await?; + let provider_handle = app_state.active_provider.acquire_connection_with_lease_for_session( + &input.name, + &fingerprint.addr, + false, + connection_priority_for_kind(user, connection_kind), + connection_kind, + Some(PlaybackLeaseRef::new(session_owner, playback_kind)), + )?; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); return None; }; let Some((_provider_name, stream_url)) = select_provider_stream_url(request_url, input, &provider_config, false, &app_state.app_config).await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); return None; }; @@ -691,10 +689,12 @@ pub(super) async fn try_reserve_hls_entry_origin_account_for_redirect( user_session_token.to_string() }; - app_state - .active_provider - .refresh_provider_reservation(&provider_config.name, session_owner, reservation_ttl_secs) - .await; + app_state.active_provider.refresh_adaptive_playback_lease( + &provider_config.name, + session_owner, + playback_kind, + reservation_ttl_secs, + ); Some(HlsEntryOriginAccountReservation { request_url: stream_url, @@ -728,7 +728,7 @@ pub(super) async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( false, ) .await; - if connection_admission.permission == UserConnectionPermission::Exhausted { + if connection_admission.permission() == UserConnectionPermission::Exhausted { return false; } @@ -743,7 +743,7 @@ pub(super) async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( let Some(hls_cache_origin) = build_hls_origin_resolution(input, &origin_playlist_url) else { return false; }; - let Some(connection_kind) = connection_admission.kind else { + let Some(connection_kind) = connection_admission.kind() else { return false; }; let (shared_hls_session_owner, reservation_ttl_secs) = if hls_cache_enabled_for_target(app_state, target) { @@ -768,8 +768,9 @@ pub(super) async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( hls_cache_origin.session_entry_url.as_str(), &session_token, session_owner, + PlaybackKind::LiveHls, reservation_ttl_secs, - connection_admission.permission, + connection_admission.permission(), connection_kind, false, ) @@ -778,7 +779,7 @@ pub(super) async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( return false; }; - app_state.connection_manager.release_provider_handle(reservation.provider_handle).await; + app_state.connection_manager.release_provider_handle(reservation.provider_handle); true } @@ -1638,10 +1639,7 @@ pub(super) async fn try_hls_cache_canonical_manifest_response( (HlsManifestAcceptanceDirective::none(), Some(owner_key)) } HlsManifestAcceptanceEvaluationOutcome::SessionSuperseded => { - app_state - .connection_manager - .release_provider_handle(prepared_origin.preacquired_origin_account_handle) - .await; + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); return Some(hls_canonical_retry_after_response()); } }; @@ -1712,7 +1710,7 @@ pub(super) async fn try_hls_cache_canonical_manifest_response( HlsManifestRefreshOrdering::Background }; if let Some(owner_key) = availability_reevaluation_owner_key { - app_state.connection_manager.release_provider_handle(preacquired_provider_handle).await; + app_state.connection_manager.release_provider_handle(preacquired_provider_handle); touch_initial_manifest_access_lease_window( app_state, access_lease_id, diff --git a/backend/app/src/api/endpoints/hls_api/mod.rs b/backend/app/src/api/endpoints/hls_api/mod.rs index 6bd7fad18..87b54839c 100644 --- a/backend/app/src/api/endpoints/hls_api/mod.rs +++ b/backend/app/src/api/endpoints/hls_api/mod.rs @@ -29,7 +29,7 @@ use crate::{ hls_provisioning_discontinuity_sequence, hls_virtual_entry_redirect_response, is_custom_video_stream_enabled, start_hls_panel_provisioning_once, try_hls_panel_provisioning_manifest_response, AppState, ConnectionHistoryMode, CustomVideoStreamType, - GraceMode, HlsPanelProvisioningRedirectPaths, HlsProvisioningStatus, ProviderAllocation, + GraceMode, HlsPanelProvisioningRedirectPaths, HlsProvisioningStatus, PlaybackLeaseRef, ProviderAllocation, ProviderConfig as RuntimeProviderConfig, ProviderHandle, StreamMeterHandle, TransportStreamBuffer, UserSession, }, @@ -37,7 +37,7 @@ use crate::{ }, auth::{check_network_access_only, Fingerprint}, model::{ - ConfigInput, ConfigInputFlags, ConfigProvider, ConfigTarget, InputSource, ProxyUserCredentials, + ConfigInput, ConfigInputFlags, ConfigProvider, ConfigTarget, InputSource, PlaybackKind, ProxyUserCredentials, ReverseProxyDisabledHeaderConfig, }, processing::parser::hls::{ @@ -115,9 +115,9 @@ use tuliprox_hls::{ OriginSegmentKey, ProxySessionId, RetryPolicy, SegmentCacheKey, SegmentCacheStatus, SegmentDemandFetchOutcome, SegmentEntry, SegmentFetchContext, SegmentFetchPolicy, TransientManifestGeneration, TransientObjectFetchToken, TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, TransientResourceId, - TransientResourceRef, HLS_ACCESS_LEASE_ID_PLACEHOLDER, HLS_PROVISIONING_GAP_ORIGIN_EPOCH, - HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, HLS_PROVISIONING_TARGET_DURATION_SECS, - MAX_HLS_MANIFEST_BYTES, + TransientResourceKind, TransientResourceRef, HLS_ACCESS_LEASE_ID_PLACEHOLDER, + HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, + HLS_PROVISIONING_TARGET_DURATION_SECS, MAX_HLS_MANIFEST_BYTES, }, HlsCtx, MAX_MANUAL_REDIRECTS, }; diff --git a/backend/app/src/api/endpoints/hls_api/segment.rs b/backend/app/src/api/endpoints/hls_api/segment.rs index 180881e67..b533d8ca8 100644 --- a/backend/app/src/api/endpoints/hls_api/segment.rs +++ b/backend/app/src/api/endpoints/hls_api/segment.rs @@ -139,16 +139,6 @@ pub(super) async fn prepare_hls_resource_access( return Err(Box::new(hls_resource_access_lease_validation_response(&err))); } }; - app_state - .hls_proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - proxy_session_id, - now_ms, - ) - .await; reclaim_hls_account_overlap_if_needed(app_state, &session, now_ms).await; let Some(lease) = app_state.hls_proxy.access_lease_response_snapshot(&access_context.lease_id, proxy_session_id, now_ms).await @@ -219,9 +209,14 @@ pub(super) async fn hls_cache_response_context( now_ms: u64, ) -> HlsCacheResponseContext { let qos_meter = app_state.hls_proxy.qos().meter_for_access_lease(&access_context.lease_id).await; - let log_identity = { + let (log_identity, session_owner, playback_request_id) = { let session = session.read().await; - HlsLogIdentity::from_session(&session) + let binding = session.origin_account_binding.as_ref().filter(|binding| binding.is_active()); + ( + HlsLogIdentity::from_session(&session), + binding.map(|binding| binding.session_owner.clone()), + binding.and_then(|binding| binding.playback_request_id), + ) }; HlsCacheResponseContext::new( access_context.lease_id.clone(), @@ -230,13 +225,20 @@ pub(super) async fn hls_cache_response_context( Arc::clone(app_state.hls_proxy.metrics()), Arc::clone(app_state.hls_proxy.segment_repair()), qos_meter, - Some(HlsMediaActivityMarker::new( - Arc::clone(&app_state.hls_proxy), - Arc::clone(session), - access_context.proxy_session_id.clone(), - access_context.lease_id.clone(), - lease_identity, - )), + Some( + HlsMediaActivityMarker::new( + Arc::clone(&app_state.hls_proxy), + Arc::clone(session), + access_context.proxy_session_id.clone(), + access_context.lease_id.clone(), + lease_identity, + ) + .with_active_provider( + Arc::clone(&app_state.active_provider), + session_owner, + playback_request_id, + ), + ), now_ms, ) } @@ -926,6 +928,7 @@ pub(super) async fn fetch_or_passthrough_transient_resource( serve_hls_transient_passthrough_result(endpoint, resource, policy, fetch_result).await } +#[allow(clippy::too_many_lines)] pub(super) async fn serve_hls_transient_passthrough_result( endpoint: HlsResourceEndpointContext<'_>, resource: TransientResourceRef, @@ -981,7 +984,22 @@ pub(super) async fn serve_hls_transient_passthrough_result( ); } } - hls_transient_origin_response( + let media_marker = if matches!(resource.kind, TransientResourceKind::Segment | TransientResourceKind::Part) + && response.decoded.status.is_success() + { + hls_cache_response_context( + endpoint.app_state, + endpoint.session, + endpoint.access_context, + endpoint.lease_identity, + endpoint.now_ms, + ) + .await + .media_activity_marker + } else { + None + }; + let response = hls_transient_origin_response( response, HlsTransientDirectResponseContext { session: Arc::clone(endpoint.session), @@ -993,7 +1011,12 @@ pub(super) async fn serve_hls_transient_passthrough_result( HlsLogIdentity::from_session(&session) }, }, - ) + ); + if let Some(marker) = media_marker { + marker.confirm_media_response(response) + } else { + response + } } Err(err) => { if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { @@ -1977,7 +2000,7 @@ pub(super) async fn fetch_and_cache_transient_origin_response( resource_kind: context.resource.kind, clients, policy: policy.clone(), - log_identity, + log_identity: log_identity.clone(), }; let cache_fetch_request = HlsTransientOriginCacheFetchRequest { fetch: fetch_request, @@ -1985,6 +2008,8 @@ pub(super) async fn fetch_and_cache_transient_origin_response( segment_cache: Arc::clone(context.app_state.hls_proxy.segment_cache()), segment_repair: Arc::clone(context.app_state.hls_proxy.segment_repair()), session: Arc::clone(context.session), + proxy_session_id: context.access_context.proxy_session_id.clone(), + log_identity: log_identity.clone(), access_lease_id: context.access_context.lease_id.clone(), resource: context.resource.clone(), resource_file: context.resource_file.clone(), @@ -2120,7 +2145,7 @@ pub(super) async fn terminate_failed_hls_manifest_session( ) { let _transition_guard = app_state.active_users.acquire_playback_transition(username, session_token).await; app_state.active_users.terminate_session(username, session_token).await; - app_state.active_provider.clear_provider_reservation(session_token).await; + app_state.active_provider.clear_provider_reservation(session_token); } pub(super) fn normalize_xtream_live_hls_url(hls_url: &str, input: &ConfigInput) -> String { @@ -2334,9 +2359,9 @@ pub(in crate::api) async fn handle_hls_stream_request( let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); let (request_url, session_token, provider_handle, _selected_provider_config) = if let Some(session) = user_session { let pinned_provider = if session.provider.is_empty() { &input.name } else { &session.provider }; - let provider_handle = if let Some(handle) = app_state - .active_provider - .acquire_exact_connection_with_grace_for_session( + let pinned_kind = if archive_reference.is_some() { PlaybackKind::Catchup } else { PlaybackKind::LiveHls }; + let provider_handle = if let Some(handle) = + app_state.active_provider.acquire_exact_connection_with_lease_for_session( pinned_provider, &fingerprint.addr, false, @@ -2345,10 +2370,8 @@ pub(in crate::api) async fn handle_hls_stream_request( session.connection_kind.or(connection_kind).unwrap_or(crate::api::model::ConnectionKind::Normal), ), session.connection_kind.or(connection_kind).unwrap_or(crate::api::model::ConnectionKind::Normal), - Some(session.token.as_str()), - ) - .await - { + Some(PlaybackLeaseRef::new(session.token.as_str(), pinned_kind)), + ) { Some(handle) } else { debug_if_enabled!( @@ -2378,7 +2401,7 @@ pub(in crate::api) async fn handle_hls_stream_request( let Some((_provider_name, stream_url)) = select_provider_stream_url(&url, input, cfg, false, &app_state.app_config).await else { - app_state.connection_manager.release_provider_handle(provider_handle).await; + app_state.connection_manager.release_provider_handle(provider_handle); return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); }; let session_token = app_state @@ -2396,10 +2419,12 @@ pub(in crate::api) async fn handle_hls_stream_request( }) .await; let hls_session_ttl_secs = get_hls_session_ttl_secs(app_state); - app_state - .active_provider - .refresh_provider_reservation(&cfg.name, &session_token, hls_session_ttl_secs) - .await; + app_state.active_provider.refresh_adaptive_playback_lease( + &cfg.name, + &session_token, + pinned_kind, + hls_session_ttl_secs, + ); (stream_url, Some(session_token), provider_handle, Some(selected_provider_config)) } None => (url, None, None, None), @@ -2423,6 +2448,9 @@ pub(in crate::api) async fn handle_hls_stream_request( }; let session_owner = hls_session_owner.as_deref().unwrap_or(user_session_token.as_str()); let hls_session_ttl_secs = get_hls_session_ttl_secs(app_state); + // Archive playback keeps its own reconnect window semantics, so the lease must + // be classified as catchup rather than plain live HLS. + let playback_kind = if archive_reference.is_some() { PlaybackKind::Catchup } else { PlaybackKind::LiveHls }; let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( app_state, fingerprint, @@ -2432,6 +2460,7 @@ pub(in crate::api) async fn handle_hls_stream_request( &url, &user_session_token, session_owner, + playback_kind, hls_session_ttl_secs, connection_permission, fallback_connection_kind, @@ -2485,7 +2514,7 @@ pub(in crate::api) async fn handle_hls_stream_request( // Playlist requests only need the chosen provider account to derive the URL and pin the session. // Holding the provider slot until the first segment request causes stale active connections and // breaks forced same-account reuse on the next HLS/Catchup stream request. - app_state.connection_manager.release_provider_handle(provider_handle).await; + app_state.connection_manager.release_provider_handle(provider_handle); let input_source = InputSource::from(input).with_url(request_url); let download_result = download_legacy_hls_manifest(app_state, &input_source, &headers).await; @@ -2862,7 +2891,7 @@ pub(super) async fn hls_api_stream_resolved( .await; } - if app_state.active_provider.is_over_limit(&session.provider).await { + if app_state.active_provider.is_over_limit(&session.provider) { let stream_channel = resolve_stream_channel( &app_state, &target, @@ -2946,8 +2975,8 @@ pub(super) async fn hls_api_stream_resolved( false, ) .await; - let connection_permission = connection_admission.permission; - let connection_kind = connection_admission.kind.or(session.connection_kind); + let connection_permission = connection_admission.permission(); + let connection_kind = connection_admission.kind().or(session.connection_kind); session.permission = connection_permission; if let Some(connection_kind) = connection_kind { session.connection_kind = Some(connection_kind); @@ -2955,6 +2984,9 @@ pub(super) async fn hls_api_stream_resolved( if connection_permission == UserConnectionPermission::Exhausted || (connection_permission == UserConnectionPermission::GracePeriod && connection_kind.is_none()) { + if connection_admission.is_reentry_suppressed() { + return crate::api::api_utils::reentry_suppressed_response(); + } let provider = if session.provider.is_empty() { input.name.clone() } else { session.provider.clone() }; let stream_channel = resolve_stream_channel( &app_state, diff --git a/backend/app/src/api/endpoints/hls_api/session.rs b/backend/app/src/api/endpoints/hls_api/session.rs index 853cb01ec..edd966f16 100644 --- a/backend/app/src/api/endpoints/hls_api/session.rs +++ b/backend/app/src/api/endpoints/hls_api/session.rs @@ -1,7 +1,7 @@ #![allow(clippy::wildcard_imports)] use super::*; -use tuliprox_core::model::AppConfig; +use tuliprox_core::model::{AppConfig, ProviderBindingTag}; #[derive(Clone, Debug)] pub(super) enum HlsOriginEntryUrl { @@ -446,6 +446,7 @@ pub(super) struct HlsAccountOverlapCandidate { pub(super) soft_overlap_delay_ms: u64, pub(super) tuliprox_target_user_connection_capacity: u32, pub(super) origin_input_account_connection_capacity: u32, + pub(super) provider_binding_tag: Option, } #[derive(Clone)] @@ -457,12 +458,15 @@ pub(super) struct HlsOriginPolicyPreemptCandidate { pub(super) reservation_ttl_secs: u64, pub(super) victim_policy: HlsEffectiveOriginAcquirePolicy, pub(super) last_media_at_ms: u64, + pub(super) provider_binding_tag: Option, } trait HlsOriginReservationCandidate { fn account_name(&self) -> &Arc; fn session_owner(&self) -> &str; fn reservation_ttl_secs(&self) -> u64; + fn proxy_session_id(&self) -> &ProxySessionId; + fn cleared_binding_tag(&self) -> Option; } impl HlsOriginReservationCandidate for HlsAccountOverlapCandidate { @@ -471,6 +475,10 @@ impl HlsOriginReservationCandidate for HlsAccountOverlapCandidate { fn session_owner(&self) -> &str { &self.session_owner } fn reservation_ttl_secs(&self) -> u64 { self.reservation_ttl_secs } + + fn proxy_session_id(&self) -> &ProxySessionId { &self.proxy_session_id } + + fn cleared_binding_tag(&self) -> Option { self.provider_binding_tag } } impl HlsOriginReservationCandidate for HlsOriginPolicyPreemptCandidate { @@ -479,6 +487,10 @@ impl HlsOriginReservationCandidate for HlsOriginPolicyPreemptCandidate { fn session_owner(&self) -> &str { &self.session_owner } fn reservation_ttl_secs(&self) -> u64 { self.reservation_ttl_secs } + + fn proxy_session_id(&self) -> &ProxySessionId { &self.proxy_session_id } + + fn cleared_binding_tag(&self) -> Option { self.provider_binding_tag } } #[derive(Debug, Clone, Copy)] @@ -505,11 +517,8 @@ pub(super) fn hls_soft_overlap_capacity_for_target_duration( } } -pub(super) async fn hls_origin_input_account_connection_capacity( - app_state: &Arc, - input: &ConfigInput, -) -> u32 { - let capacities = app_state.active_provider.provider_capacities_for_input(&input.name).await; +pub(super) fn hls_origin_input_account_connection_capacity(app_state: &Arc, input: &ConfigInput) -> u32 { + let capacities = app_state.active_provider.provider_capacities_for_input(&input.name); if capacities.is_empty() { return hls_configured_origin_input_account_connection_capacity(input); } @@ -613,6 +622,18 @@ pub(super) fn hls_soft_overlap_delay_ms( target_duration_ms.saturating_mul(numerator).saturating_add(origin.saturating_sub(1)) / origin } +/// Playback kind used for provider leases of a shared HLS session. +/// +/// Archive-backed sessions must keep catchup semantics instead of plain live HLS; +/// live sessions keep the live HLS kind. +async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind { + if session.read().await.key.archive_reference.is_some() { + PlaybackKind::Catchup + } else { + PlaybackKind::LiveHls + } +} + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(super) async fn prepare_hls_origin_runtime( app_state: &Arc, @@ -628,6 +649,7 @@ pub(super) async fn prepare_hls_origin_runtime( work_class: HlsOriginWorkClass, now_ms: u64, ) -> Result { + let playback_kind = hls_session_origin_playback_kind(session).await; promote_elapsed_hls_account_overlaps(app_state, now_ms).await; detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await; reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await; @@ -652,6 +674,7 @@ pub(super) async fn prepare_hls_origin_runtime( fingerprint, connection_kind, priority, + playback_kind, now_ms, ) .await; @@ -682,6 +705,7 @@ pub(super) async fn prepare_hls_origin_runtime( false, work_kind, work_class, + playback_kind, now_ms, ) .await @@ -709,6 +733,7 @@ pub(super) async fn prepare_hls_origin_runtime( fingerprint, connection_kind, priority, + playback_kind, now_ms, ) .await @@ -731,6 +756,7 @@ pub(super) async fn prepare_hls_origin_runtime( fingerprint, connection_kind, priority, + playback_kind, now_ms, ) .await @@ -759,6 +785,7 @@ pub(super) async fn prepare_hls_origin_runtime( true, work_kind, work_class, + playback_kind, now_ms, ) .await @@ -802,21 +829,18 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( allow_grace: bool, work_kind: HlsOriginWorkKind, work_class: HlsOriginWorkClass, + playback_kind: PlaybackKind, now_ms: u64, ) -> Result { let session_owner = build_hls_origin_session_owner(proxy_session_id); - let Some(provider_handle) = app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &fingerprint.addr, - allow_grace, - priority, - connection_kind, - Some(&session_owner), - ) - .await - else { + let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( + &input.name, + &fingerprint.addr, + allow_grace, + priority, + connection_kind, + Some(PlaybackLeaseRef::new(&session_owner, playback_kind)), + ) else { debug!( "HLS origin account acquire unavailable: work={} work_class={} grace={}", work_kind.as_log_value(), @@ -829,7 +853,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -842,7 +866,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -852,7 +876,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( &provider_handle.allocation, now_ms, ) else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -970,8 +994,16 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A protection.as_log_state(), timing.target_duration_ms ); - if !matches!(protection, HlsAccountBindingProtection::Expired) - || session_guard.activity.active_origin_work_count > 0 + let starting_window_ms = tuliprox_session::STARTING_LEASE_TTL_SECS.saturating_mul(1000); + let media_window_expired = session_guard.activity.last_delivered_media_at_ms.map_or_else( + || now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms), + |last| { + now_ms + > last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms) + }, + ); + if !media_window_expired + || session_guard.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { continue; } @@ -990,7 +1022,11 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A ); binding }; - app_state.active_provider.clear_provider_reservation(&binding.session_owner).await; + app_state.active_provider.clear_identified_provider_reservation( + &binding.session_owner, + &binding.account_name, + binding.provider_binding_tag, + ); } } @@ -1005,6 +1041,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( fingerprint: &Fingerprint, connection_kind: crate::api::model::ConnectionKind, priority: i8, + playback_kind: PlaybackKind, now_ms: u64, ) -> Result { let request_policy = HlsEffectiveOriginAcquirePolicy::new(connection_kind, priority, now_ms); @@ -1015,27 +1052,44 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; - app_state.active_provider.clear_provider_reservation(&candidate.session_owner).await; + let is_valid_victim = { + let victim = candidate.session.read().await; + let has_no_active_origin_work = + victim.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) == 0; + victim.origin_account_binding.as_ref().is_some_and(|b| { + b.account_name == candidate.account_name + && b.session_owner == candidate.session_owner + && b.provider_binding_tag == candidate.provider_binding_tag + && matches!(b.binding_mode, HlsOriginAccountBindingMode::Active) + && has_no_active_origin_work + }) + }; + if !is_valid_victim { + debug!("HLS origin policy preemption denied: reason=stale-candidate"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + + app_state.active_provider.clear_identified_provider_reservation( + &candidate.session_owner, + &candidate.account_name, + candidate.provider_binding_tag, + ); let session_owner = build_hls_origin_session_owner(proxy_session_id); - let Some(provider_handle) = app_state - .active_provider - .acquire_exact_connection_with_grace_for_session( - &candidate.account_name, - &fingerprint.addr, - false, - priority, - connection_kind, - Some(&session_owner), - ) - .await - else { - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + let Some(provider_handle) = app_state.active_provider.acquire_exact_connection_with_lease_for_session( + &candidate.account_name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(PlaybackLeaseRef::new(&session_owner, playback_kind)), + ) else { + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=exact-acquire-failed"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=missing-provider-config"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -1048,8 +1102,8 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=invalid-origin-url"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -1059,8 +1113,8 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( &provider_handle.allocation, now_ms, ) else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=invalid-allocation"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -1068,10 +1122,12 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( let mut detached_victim = false; { let mut victim = candidate.session.write().await; - let has_no_active_origin_work = victim.activity.active_origin_work_count == 0; + let has_no_active_origin_work = + victim.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) == 0; if let Some(victim_binding) = victim.origin_account_binding.as_mut() { if victim_binding.account_name == candidate.account_name && victim_binding.session_owner == candidate.session_owner + && victim_binding.provider_binding_tag == candidate.provider_binding_tag && matches!(victim_binding.binding_mode, HlsOriginAccountBindingMode::Active) && has_no_active_origin_work { @@ -1084,8 +1140,8 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( } } if !detached_victim { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=stale-candidate"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); } @@ -1125,18 +1181,39 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( }) } +/// Restores a victim reservation that was cleared for a preemption that did not complete. +/// +/// The lease is recreated rather than refreshed, because the clear already removed it and +/// a plain refresh would be a no-op. The victim binding's tag is updated to the recreated +/// incarnation so a later clear still matches it. async fn restore_hls_origin_policy_preempt_candidate_reservation( app_state: &Arc, candidate: &C, + playback_kind: PlaybackKind, ) { - app_state - .active_provider - .refresh_provider_reservation( - candidate.account_name(), - candidate.session_owner(), - candidate.reservation_ttl_secs(), - ) - .await; + let account_name = candidate.account_name(); + let session_owner = candidate.session_owner(); + let cleared_tag = candidate.cleared_binding_tag(); + app_state.active_provider.refresh_identified_provider_reservation( + account_name, + session_owner, + playback_kind, + candidate.reservation_ttl_secs(), + ); + let new_tag = app_state.active_provider.binding_tag_for_owner(session_owner); + let Some(session) = app_state.hls_proxy.sessions().get_by_proxy_session_id(candidate.proxy_session_id()).await + else { + return; + }; + let mut session_guard = session.write().await; + if let Some(binding) = session_guard.origin_account_binding.as_mut() { + if binding.account_name == *account_name + && binding.session_owner == session_owner + && binding.provider_binding_tag == cleared_tag + { + binding.provider_binding_tag = new_tag; + } + } } pub(super) async fn find_hls_origin_policy_preempt_candidate( @@ -1159,7 +1236,7 @@ pub(super) async fn find_hls_origin_policy_preempt_candidate( if binding.input_name != input.name || !matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active) { continue; } - if session_guard.activity.active_origin_work_count > 0 { + if session_guard.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { continue; } if !matches!(hls_origin_account_status(&app_state.hls_ctx(), binding), HlsOriginAccountStatus::Known) { @@ -1177,6 +1254,7 @@ pub(super) async fn find_hls_origin_policy_preempt_candidate( reservation_ttl_secs: session_guard.account_overlap_timing().reservation_ttl_secs(), victim_policy, last_media_at_ms: session_guard.activity.last_authorized_media_at_ms.unwrap_or_default(), + provider_binding_tag: binding.provider_binding_tag, }; if hls_origin_policy_preempt_candidate_is_better(best_candidate.as_ref(), &candidate) { best_candidate = Some(candidate); @@ -1213,33 +1291,34 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( fingerprint: &Fingerprint, connection_kind: crate::api::model::ConnectionKind, priority: i8, + playback_kind: PlaybackKind, now_ms: u64, ) -> Result { let Some(candidate) = find_hls_account_overlap_candidate(app_state, input, proxy_session_id, now_ms).await else { debug!("HLS account overlap denied: reason=no-soft-active-candidate"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; - app_state.active_provider.clear_provider_reservation(&candidate.session_owner).await; + app_state.active_provider.clear_identified_provider_reservation( + &candidate.session_owner, + &candidate.account_name, + candidate.provider_binding_tag, + ); let session_owner = build_hls_origin_session_owner(proxy_session_id); - let Some(provider_handle) = app_state - .active_provider - .acquire_exact_connection_with_grace_for_session( - &candidate.account_name, - &fingerprint.addr, - false, - priority, - connection_kind, - Some(&session_owner), - ) - .await - else { - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + let Some(provider_handle) = app_state.active_provider.acquire_exact_connection_with_lease_for_session( + &candidate.account_name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(PlaybackLeaseRef::new(&session_owner, playback_kind)), + ) else { + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS account overlap denied: reason=speculative-acquire-failed"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS account overlap denied: reason=missing-provider-config"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -1252,8 +1331,8 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; - restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS account overlap denied: reason=invalid-origin-url"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -1302,7 +1381,7 @@ pub(super) async fn find_hls_account_overlap_candidate( ) -> Option { let sessions = app_state.hls_proxy.sessions().list_sessions().await; let tuliprox_target_user_connection_capacity = hls_tuliprox_target_user_connection_capacity(app_state, input).await; - let origin_input_account_connection_capacity = hls_origin_input_account_connection_capacity(app_state, input).await; + let origin_input_account_connection_capacity = hls_origin_input_account_connection_capacity(app_state, input); let mut speculative_accounts = Vec::new(); for session in &sessions { let session = session.read().await; @@ -1337,7 +1416,7 @@ pub(super) async fn find_hls_account_overlap_candidate( if !matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active) { continue; } - if session_guard.activity.active_origin_work_count > 0 { + if session_guard.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { continue; } let timing = session_guard.account_overlap_timing(); @@ -1383,6 +1462,7 @@ pub(super) async fn find_hls_account_overlap_candidate( soft_overlap_delay_ms: capacity.delay_ms, tuliprox_target_user_connection_capacity: capacity.tuliprox_target_user_connection_capacity, origin_input_account_connection_capacity: capacity.origin_input_account_connection_capacity, + provider_binding_tag: binding.provider_binding_tag, }); } let mut eligible = filter_hls_account_overlap_cooldowns(app_state, candidates, now_ms).await; @@ -1437,9 +1517,30 @@ pub(super) async fn reclaim_hls_account_overlap_if_needed( } (session_guard.proxy_session_id.clone(), binding) }; - app_state.active_provider.clear_provider_reservation(&loser_binding.session_owner).await; + app_state.active_provider.clear_identified_provider_reservation( + &loser_binding.session_owner, + &loser_binding.account_name, + loser_binding.provider_binding_tag, + ); { let mut loser = session.write().await; + // The loser snapshot was taken before the locks were released; a successor + // binding may have been acquired since. Only detach the exact incarnation + // that was reclaimed, never a rebound successor. + let current_matches = loser.origin_account_binding.as_ref().is_some_and(|current| { + current.provider_binding_tag == loser_binding.provider_binding_tag + && current.session_owner == loser_binding.session_owner + && current.account_name == loser_binding.account_name + && matches!(¤t.binding_mode, HlsOriginAccountBindingMode::Speculative { .. }) + }); + if !current_matches { + debug!( + "HLS account overlap reclaim skipped: loser binding changed proxy_session={} account={}", + safe_proxy_session_id(&loser_proxy_session_id), + sanitize_sensitive_info(loser_binding.account_name.as_ref()) + ); + continue; + } if let Some(binding) = loser.origin_account_binding.as_mut() { binding.detach(HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, now_ms); } @@ -1517,6 +1618,11 @@ pub(super) async fn promote_elapsed_hls_account_overlaps(app_state: &Arc Result { { @@ -1578,11 +1685,17 @@ pub(super) async fn rebind_hls_origin_account( safe_proxy_session, sanitize_sensitive_info(stale_binding.account_name.as_ref()) ); - app_state.active_provider.clear_provider_reservation(&stale_binding.session_owner).await; + app_state.active_provider.clear_identified_provider_reservation( + &stale_binding.session_owner, + &stale_binding.account_name, + stale_binding.provider_binding_tag, + ); { let mut session_guard = session.write().await; if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| { - binding.account_name == stale_binding.account_name && binding.session_owner == stale_binding.session_owner + binding.account_name == stale_binding.account_name + && binding.session_owner == stale_binding.session_owner + && binding.provider_binding_tag == stale_binding.provider_binding_tag }) { binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms); debug!( @@ -1595,18 +1708,14 @@ pub(super) async fn rebind_hls_origin_account( session_guard.invalidate_queued_origin_work(); } - let Some(provider_handle) = app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &fingerprint.addr, - false, - priority, - connection_kind, - Some(&stale_binding.session_owner), - ) - .await - else { + let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( + &input.name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)), + ) else { mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await; return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: hls_no_account_reason_for_binding(Some(stale_binding)), @@ -1614,7 +1723,7 @@ pub(super) async fn rebind_hls_origin_account( }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await; return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -1628,13 +1737,13 @@ pub(super) async fn rebind_hls_origin_account( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await; return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; let Some(new_account_name) = provider_handle.allocation.get_provider_name() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await; return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -1912,7 +2021,7 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work( if !hls_origin_binding_needs_reacquire(session).await { return Ok(None); } - if session.read().await.activity.active_origin_work_count > 0 { + if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); } let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) @@ -1967,7 +2076,6 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work preacquired_provider_handle: None, started_generation: None, }; - let started_generation = session.write().await.start_origin_work(); if let Ok(lease_guard) = begin_hls_origin_account_io_bounded( &origin_io, session, @@ -1976,14 +2084,19 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work ) .await { + let (started_generation, active_origin_work_count) = { + let mut session = session.write().await; + let count = Arc::clone(&session.activity.active_origin_work_count); + (session.start_origin_work(), count) + }; return Ok(Some(HlsTransientOriginIoGuard::new( Arc::clone(session), + active_origin_work_count, origin_io, lease_guard, started_generation, ))); } - session.write().await.finish_origin_work(started_generation); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); } HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired => {} @@ -1996,7 +2109,7 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work { return Ok(None); } - if session.read().await.activity.active_origin_work_count > 0 { + if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); } let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) @@ -2026,7 +2139,7 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work }; let Some(binding) = session.read().await.origin_account_binding.clone().filter(HlsOriginAccountBinding::is_active) else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; let origin_io = HlsOriginIoContext { @@ -2040,7 +2153,6 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work started_generation: None, } .with_preacquired_provider_handle(provider_handle); - let started_generation = session.write().await.start_origin_work(); let Ok(lease_guard) = begin_hls_origin_account_io_bounded( &origin_io, session, @@ -2049,8 +2161,18 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work ) .await else { - session.write().await.finish_origin_work(started_generation); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; - Ok(Some(HlsTransientOriginIoGuard::new(Arc::clone(session), origin_io, lease_guard, started_generation))) + let (started_generation, active_origin_work_count) = { + let mut session = session.write().await; + let count = Arc::clone(&session.activity.active_origin_work_count); + (session.start_origin_work(), count) + }; + Ok(Some(HlsTransientOriginIoGuard::new( + Arc::clone(session), + active_origin_work_count, + origin_io, + lease_guard, + started_generation, + ))) } diff --git a/backend/app/src/api/endpoints/hls_api/tests.rs b/backend/app/src/api/endpoints/hls_api/tests.rs index 99a2ab224..f90e9ac56 100644 --- a/backend/app/src/api/endpoints/hls_api/tests.rs +++ b/backend/app/src/api/endpoints/hls_api/tests.rs @@ -2420,7 +2420,7 @@ async fn hls_terminal_response_serves_prepared_finite_full_and_range_bytes_per_i let segment_zero_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.ts"); let segment_one_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/1.ts"); let repair_before = app_state.hls_proxy.segment_repair().stats().await; - let provider_connections_before = app_state.active_provider.get_provider_connections_count().await; + let provider_connections_before = app_state.active_provider.get_provider_connections_count(); let head_content_length = terminal_head_content_length(&app_state, &proxy_session_id, &segment_zero_uri).await; @@ -2444,7 +2444,7 @@ async fn hls_terminal_response_serves_prepared_finite_full_and_range_bytes_per_i assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); assert_eq!(app_state.hls_proxy.segment_repair().stats().await, repair_before); - assert_eq!(app_state.active_provider.get_provider_connections_count().await, provider_connections_before); + assert_eq!(app_state.active_provider.get_provider_connections_count(), provider_connections_before); let segment_zero_again = response_body(get_response(Arc::clone(&app_state), &segment_zero_uri, None).await).await; let segment_one = response_body(get_response(Arc::clone(&app_state), &segment_one_uri, None).await).await; @@ -3540,7 +3540,7 @@ fn hls_runtime_origin_fetch_url_uses_selected_provider_account() { }; let provider = Arc::new(RuntimeProviderConfig::new( &provider_input, - Arc::new(tokio::sync::RwLock::new(ProviderConfigConnection::default())), + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), Arc::new(|_, _| {}), )); @@ -4153,7 +4153,7 @@ fn test_app_state_with_hls_proxy_and_inputs( let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(&app_config, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let config = app_config.config.load(); @@ -4329,7 +4329,14 @@ async fn hls_origin_account_io_lease_allows_parallel_same_session_origin_work() .expect("second same-session origin io joins session lease"); wait_for_provider_connection_count(&app_state, 1).await; assert_eq!( - session.read().await.origin_account_io_lease.as_ref().expect("session provider lease exists").active_io_count, + session + .read() + .await + .origin_account_io_lease + .as_ref() + .expect("session provider lease exists") + .active_io_count + .load(std::sync::atomic::Ordering::Relaxed), 2 ); @@ -4342,18 +4349,16 @@ async fn hls_origin_account_io_lease_allows_parallel_same_session_origin_work() .origin_account_io_lease .as_ref() .expect("session provider lease remains while second io is active") - .active_io_count, + .active_io_count + .load(std::sync::atomic::Ordering::Relaxed), 1 ); finish_hls_origin_account_io(&origin_io, &session, second, true).await; wait_for_provider_connection_count(&app_state, 0).await; - assert!( - app_state - .active_provider - .is_provider_reserved_for_other_session(&binding.account_name, Some("other-hls-session")) - .await - ); + assert!(!app_state + .active_provider + .is_provider_reserved_for_other_session(&binding.account_name, Some("other-hls-session"))); } #[tokio::test] @@ -4505,11 +4510,12 @@ async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_authorized_manifest_work() { let input = overlap_provider_input(); let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let session = create_bound_hls_test_session(&app_state, &input, "12345", input.name.as_ref(), 1_000).await; { let mut session = session.write().await; session.target_duration = Some(1); session.mark_authorized_media_access(1_000); + session.activity.last_delivered_media_at_ms = Some(1_000); } let old_generation = session.read().await.activity.origin_work_generation; @@ -4548,7 +4554,7 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho assert_eq!(binding.account_name.as_ref(), "account-a"); assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -4561,12 +4567,16 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work( let mut session = old_session.write().await; session.target_duration = Some(10); session.mark_authorized_media_access(1_000); + session.activity.last_delivered_media_at_ms = Some(1_000); } let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); - app_state - .active_provider - .refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60) - .await; + app_state.active_provider.refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&old_binding.session_owner); + // The binding acquired a reservation without a handle; capture its tag so the + // preempt path can prove it still targets this exact incarnation. + let binding_tag = app_state.active_provider.binding_tag_for_owner(&old_binding.session_owner); + old_session.write().await.origin_account_binding.as_mut().expect("binding exists").provider_binding_tag = + binding_tag; let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); @@ -4602,7 +4612,7 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work( Some(super::ProviderAllocation::Available(_)) )); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -4614,6 +4624,7 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { let mut session = soft_session.write().await; session.target_duration = Some(10); session.mark_authorized_media_access(10_000); + session.activity.last_delivered_media_at_ms = Some(10_000); session.reconcile_effective_origin_acquire_policy( Some(HlsEffectiveOriginAcquirePolicy::new(ConnectionKind::Soft, 0, 10_000)), 10_000, @@ -4621,10 +4632,13 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { session.activity.origin_work_generation }; let soft_binding = soft_session.read().await.origin_account_binding.clone().expect("soft binding exists"); - app_state - .active_provider - .refresh_provider_reservation(&soft_binding.account_name, &soft_binding.session_owner, 60) - .await; + app_state.active_provider.refresh_provider_reservation(&soft_binding.account_name, &soft_binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&soft_binding.session_owner); + // The binding acquired a reservation without a handle; capture its tag so the + // preempt path can prove it still targets this exact incarnation. + let binding_tag = app_state.active_provider.binding_tag_for_owner(&soft_binding.session_owner); + soft_session.write().await.origin_account_binding.as_mut().expect("binding exists").provider_binding_tag = + binding_tag; let normal_session = create_unbound_hls_test_session(&app_state, &input, "normal", 10_500).await; let normal_proxy_session_id = normal_session.read().await.proxy_session_id.clone(); @@ -4666,7 +4680,7 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { )); drop(soft_session); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -4746,10 +4760,12 @@ async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal ); } let normal_binding = normal_session.read().await.origin_account_binding.clone().expect("normal binding exists"); - app_state - .active_provider - .refresh_provider_reservation(&normal_binding.account_name, &normal_binding.session_owner, 60) - .await; + app_state.active_provider.refresh_provider_reservation( + &normal_binding.account_name, + &normal_binding.session_owner, + 60, + ); + app_state.active_provider.confirm_playback_activity(&normal_binding.session_owner); let soft_session = create_unbound_hls_test_session(&app_state, &input, "soft", 10_500).await; let soft_proxy_session_id = soft_session.read().await.proxy_session_id.clone(); @@ -4763,6 +4779,7 @@ async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal &test_fingerprint_with_addr(test_addr_with_port(55232)), ConnectionKind::Soft, -100, + crate::model::PlaybackKind::LiveHls, 10_500, ) .await; @@ -4772,12 +4789,9 @@ async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal normal_session.read().await.origin_account_binding.as_ref().map(|binding| &binding.binding_mode), Some(HlsOriginAccountBindingMode::Active) )); - assert!( - app_state - .active_provider - .is_provider_reserved_for_other_session(&normal_binding.account_name, Some("unrelated-session")) - .await - ); + assert!(app_state + .active_provider + .is_provider_reserved_for_other_session(&normal_binding.account_name, Some("unrelated-session"))); } #[tokio::test] @@ -4794,7 +4808,6 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai ConnectionKind::Normal, Some("external-owner"), ) - .await .expect("test should occupy the only provider account"); let session = create_unbound_hls_test_session(&app_state, &input, "12345", 2_000).await; let proxy_session_id = session.read().await.proxy_session_id.clone(); @@ -4830,8 +4843,8 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai input.name.as_ref() ); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; - app_state.connection_manager.release_provider_handle(Some(occupied)).await; + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_provider_handle(Some(occupied)); } #[tokio::test] @@ -5005,9 +5018,11 @@ async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() { #[tokio::test] async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segments() { let input = single_hls_provider_input("shared-provisioning-timeline-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); enable_hls_provisioning_custom_response(&app_state); - let session = create_unbound_hls_test_session(&app_state, &input, "12345", 1_000).await; + let session = create_unbound_hls_test_session(&app_state, &input, "shared-prov-timeline-12345", 1_000).await; let access_lease_id = HlsAccessLeaseId("timeline-lease".to_string()); let proxy_session_id = session.read().await.proxy_session_id.clone(); activate_test_hls_access_lease( @@ -5072,9 +5087,11 @@ async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segmen #[tokio::test] async fn stale_provisioning_segments_do_not_trigger_canonical_handoff() { let input = single_hls_provider_input("stale-provisioning-handoff-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); enable_hls_provisioning_custom_response(&app_state); - let session = create_unbound_hls_test_session(&app_state, &input, "12345", 1_000).await; + let session = create_unbound_hls_test_session(&app_state, &input, "54321", 1_000).await; let initial_lease_id = HlsAccessLeaseId("initial-provisioning-lease".to_string()); let strip = app_state.hls_proxy.strip(); @@ -5100,7 +5117,7 @@ async fn stale_provisioning_segments_do_not_trigger_canonical_handoff() { &app_state, &session, &input, - 12345, + 54321, &new_lease_id, 2_000, ) @@ -5132,9 +5149,11 @@ async fn provisioning_handoff_finds_shared_session_by_input_stream_id_not_virtua #[tokio::test] async fn shared_provisioning_handoff_continues_proxy_sequence_for_origin_segments() { let input = single_hls_provider_input("shared-provisioning-handoff-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); enable_hls_provisioning_custom_response(&app_state); - let session = create_unbound_hls_test_session(&app_state, &input, "12345", 1_000).await; + let session = create_unbound_hls_test_session(&app_state, &input, "shared-handoff-12345", 1_000).await; let access_lease_id = HlsAccessLeaseId("handoff-lease".to_string()); let strip = app_state.hls_proxy.strip(); super::hls_shared_provisioning_timeline_manifest_response( @@ -5204,12 +5223,11 @@ async fn hls_origin_runtime_background_skips_soft_overlap_and_grace() { let mut session = old_session.write().await; session.target_duration = Some(10); session.mark_authorized_media_access(1_000); + session.activity.last_delivered_media_at_ms = Some(1_000); } let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); - app_state - .active_provider - .refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60) - .await; + app_state.active_provider.refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&old_binding.session_owner); let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); @@ -5270,7 +5288,7 @@ fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_ } #[tokio::test] -async fn hls_account_binding_without_media_activity_is_not_detached() { +async fn hls_account_binding_without_media_activity_expires_after_startup_window() { let input = overlap_provider_input(); let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; @@ -5281,12 +5299,17 @@ async fn hls_account_binding_without_media_activity_is_not_detached() { } let old_generation = session.read().await.activity.origin_work_generation; + super::detach_unprotected_hls_origin_account_bindings(&app_state, 5_000).await; + assert!(session.read().await.origin_account_binding.as_ref().is_some_and(|binding| binding.is_active())); super::detach_unprotected_hls_origin_account_bindings(&app_state, 60_000).await; let session = session.read().await; let binding = session.origin_account_binding.as_ref().expect("binding remains"); - assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); - assert_eq!(session.activity.origin_work_generation, old_generation); + assert!(matches!( + binding.binding_mode, + HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } + )); + assert_eq!(session.activity.origin_work_generation, old_generation + 1); assert_eq!(session.activity.last_authorized_media_at_ms, None); assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); } @@ -5561,8 +5584,9 @@ async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues let proxy_session_id = session.read().await.proxy_session_id.clone(); let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); let account_name = Arc::from("account-a"); - app_state.active_provider.refresh_provider_reservation(&account_name, &binding.session_owner, 60).await; - assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner")).await); + app_state.active_provider.refresh_provider_reservation(&account_name, &binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&binding.session_owner); + assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); activate_test_hls_access_lease(&app_state, &proxy_session_id, "detach-lease", 1_000, 1_000).await; register_test_hls_stream_for_lease_release(&app_state, &session, &proxy_session_id, input.name.as_ref()).await; assert_eq!(app_state.active_users.active_streams().await.len(), 1); @@ -5633,7 +5657,7 @@ async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues )); assert!(matches!(session.maps.get(&ProxyMapId(1)).expect("map remains").status, MapCacheStatus::Queued { .. })); } - assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner")).await); + assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); assert!(app_state.active_users.active_streams().await.is_empty()); } @@ -7954,6 +7978,7 @@ async fn hls_entry_origin_reservation_requires_real_provider_handle() { "http://origin.example.com/live/source-user/source-pass/12345.m3u8", "hls-session-token", "hls-cache:test-session", + crate::model::PlaybackKind::LiveHls, super::hls_origin_account_reservation_ttl_secs_fallback(), UserConnectionPermission::Allowed, ConnectionKind::Normal, @@ -7965,7 +7990,7 @@ async fn hls_entry_origin_reservation_requires_real_provider_handle() { } #[tokio::test] -async fn hls_entry_origin_reservation_sets_owner_reservation_before_redirect() { +async fn hls_entry_origin_reservation_blocks_foreign_sessions_only_after_media_confirmation() { let input = single_hls_provider_input("available-provider"); let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); let mut user = ProxyUserCredentials::default(); @@ -7981,6 +8006,7 @@ async fn hls_entry_origin_reservation_sets_owner_reservation_before_redirect() { "http://account.example.com/live/account-user/account-pass/12345.m3u8", "hls-session-token", session_owner, + crate::model::PlaybackKind::LiveHls, super::hls_origin_account_reservation_ttl_secs_fallback(), UserConnectionPermission::Allowed, ConnectionKind::Normal, @@ -7992,37 +8018,50 @@ async fn hls_entry_origin_reservation_sets_owner_reservation_before_redirect() { assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); assert!(reservation.selected_provider_config.is_some()); assert!(app_state.active_users.active_streams().await.is_empty()); - app_state.connection_manager.release_provider_handle(reservation.provider_handle).await; + app_state.connection_manager.release_provider_handle(reservation.provider_handle); + // A manifest-only entry reservation is still unconfirmed, so it must not block a + // different HLS session: an abandoned start or a manifest retry has to leave the + // provider capacity free for unrelated clients behind the same reverse proxy. + let other_handle = app_state.active_provider.acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55251), + false, + 0, + ConnectionKind::Normal, + Some("other-owner"), + ); + assert!(other_handle.is_some(), "an unconfirmed entry reservation must not block other HLS sessions"); + app_state.connection_manager.release_provider_handle(other_handle); + + let same_owner_handle = app_state.active_provider.acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55252), + false, + 0, + ConnectionKind::Normal, + Some(session_owner), + ); + assert!(same_owner_handle.is_some(), "reserved provider must be reusable by the same HLS session owner"); + app_state.connection_manager.release_provider_handle(same_owner_handle); + + // Once real media activity confirms the lease, it reserves the provider slot and + // other sessions are routed away from it. + app_state.active_provider.confirm_playback_activity(session_owner); assert!( app_state .active_provider .acquire_connection_with_grace_for_session( &input.name, - &test_addr_with_port(55251), + &test_addr_with_port(55253), false, 0, ConnectionKind::Normal, Some("other-owner"), ) - .await .is_none(), - "reserved provider must stay blocked for other HLS sessions" + "a confirmed reservation must stay blocked for other HLS sessions" ); - - let same_owner_handle = app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &test_addr_with_port(55252), - false, - 0, - ConnectionKind::Normal, - Some(session_owner), - ) - .await; - assert!(same_owner_handle.is_some(), "reserved provider must be reusable by the same HLS session owner"); - app_state.connection_manager.release_provider_handle(same_owner_handle).await; } #[tokio::test] @@ -8091,7 +8130,6 @@ async fn hls_entry_origin_reservation_uses_persisted_alias_manifest_url() { ConnectionKind::Normal, Some("primary-session"), ) - .await .expect("primary account should be allocated"); let reservation = super::try_reserve_hls_entry_origin_account_for_redirect( @@ -8107,6 +8145,7 @@ async fn hls_entry_origin_reservation_uses_persisted_alias_manifest_url() { "http://stream.example:4000/news24hd/mono.m3u8?token=primary-stream-token", "alias-session-token", "alias-session-owner", + crate::model::PlaybackKind::LiveHls, super::hls_origin_account_reservation_ttl_secs_fallback(), UserConnectionPermission::Allowed, ConnectionKind::Normal, @@ -8121,8 +8160,8 @@ async fn hls_entry_origin_reservation_uses_persisted_alias_manifest_url() { ); assert_eq!(reservation.request_url, "http://stream.example:4000/news24hd/mono.m3u8?token=alias-stream-token"); - app_state.connection_manager.release_provider_handle(reservation.provider_handle).await; - app_state.connection_manager.release_provider_handle(Some(primary_handle)).await; + app_state.connection_manager.release_provider_handle(reservation.provider_handle); + app_state.connection_manager.release_provider_handle(Some(primary_handle)); } #[tokio::test] @@ -8157,19 +8196,16 @@ async fn hls_virtual_entry_reservation_uses_input_stream_id_for_shared_session_o let expected_key = HlsSessionKey::new(input.id, "80510"); let expected_proxy_session_id = build_proxy_session_id(&expected_key, &app_state.get_encrypt_secret()); let expected_owner = crate::api::model::build_hls_origin_session_owner(&expected_proxy_session_id); - let same_owner_handle = app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &test_addr_with_port(55253), - false, - 0, - ConnectionKind::Normal, - Some(&expected_owner), - ) - .await; + let same_owner_handle = app_state.active_provider.acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55253), + false, + 0, + ConnectionKind::Normal, + Some(&expected_owner), + ); assert!(same_owner_handle.is_some(), "reservation must be owned by input:1|hls|80510, not virtual_id=1001"); - app_state.connection_manager.release_provider_handle(same_owner_handle).await; + app_state.connection_manager.release_provider_handle(same_owner_handle); } #[tokio::test] @@ -10522,13 +10558,13 @@ async fn terminal_test_plan_shape(app_state: &Arc, proxy_session_id: & async fn wait_for_provider_connection_count(app_state: &Arc, expected: usize) { for _ in 0..50 { - let actual = app_state.active_provider.get_provider_connections_count().await; + let actual = app_state.active_provider.get_provider_connections_count(); if actual == expected { return; } tokio::time::sleep(Duration::from_millis(10)).await; } - assert_eq!(app_state.active_provider.get_provider_connections_count().await, expected); + assert_eq!(app_state.active_provider.get_provider_connections_count(), expected); } fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { @@ -10654,9 +10690,12 @@ async fn map_transient_resource_with_kind( grant_lease: bool, kind: TransientResourceKind, ) -> (String, String) { + static TRANSIENT_STREAM_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); let secret = b"rewrite-secret"; let now_ms = super::current_time_millis(); - let session = app_state.hls_proxy.get_or_create_session(HlsSessionKey::new(1, "12345"), secret, now_ms).await; + let stream_ref = + format!("transient-{}", TRANSIENT_STREAM_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)); + let session = app_state.hls_proxy.get_or_create_session(HlsSessionKey::new(1, &stream_ref), secret, now_ms).await; let resource_id = build_transient_resource_id(origin_url, secret); let proxy_session_id = { let mut session = session.write().await; @@ -12121,7 +12160,7 @@ async fn transient_decoder_failure_releases_origin_and_access_guards_once() { wait_for_provider_connection_count(&app_state, 0).await; assert_eq!(resource.active_readers(), 0); tokio::task::yield_now().await; - assert_eq!(app_state.active_provider.get_provider_connections_count().await, 0); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); assert_eq!(origin.requests.lock().await.len(), 1, "body failure must not start another origin request"); } diff --git a/backend/app/src/api/endpoints/m3u_api.rs b/backend/app/src/api/endpoints/m3u_api.rs index 0aa5df6a0..2a4961a08 100644 --- a/backend/app/src/api/endpoints/m3u_api.rs +++ b/backend/app/src/api/endpoints/m3u_api.rs @@ -5,8 +5,8 @@ use crate::{ create_playback_session_fingerprint, create_session_fingerprint, force_provider_stream_response, get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, local_stream_response, - redirect, redirect_response, resolve_initial_stalker_playback_url, resource_response, - separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, + redirect, redirect_response, reentry_suppressed_response, resolve_initial_stalker_playback_url, + resource_response, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, try_option_bad_request, try_result_bad_request, try_result_not_found, try_unwrap_body, RedirectParams, ResourceFetchPolicy, }, @@ -226,6 +226,9 @@ pub(in crate::api) async fn m3u_api_stream_loaded( false, ) .await; + if admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } return local_stream_response( fingerprint, app_state, @@ -234,8 +237,8 @@ pub(in crate::api) async fn m3u_api_stream_loaded( &input, &target, &user, - admission.permission, - admission.kind.unwrap_or(crate::api::model::ConnectionKind::Normal), + admission.permission(), + admission.kind().unwrap_or(crate::api::model::ConnectionKind::Normal), Some(playback_session_token.as_str()), Some(request_class), true, @@ -315,7 +318,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( ); } - if app_state.active_provider.is_over_limit(&session.provider).await { + if app_state.active_provider.is_over_limit(&session.provider) { if extension == HLS_EXT { return hls_admission_failure_manifest_response( app_state, @@ -379,9 +382,9 @@ pub(in crate::api) async fn m3u_api_stream_loaded( false, ) .await; - let connection_permission = connection_admission.permission; + let connection_permission = connection_admission.permission(); let connection_kind = connection_admission - .kind + .kind() .or(user_session.as_ref().and_then(|session| session.connection_kind)) .unwrap_or(crate::api::model::ConnectionKind::Normal); let allow_exhausted_shared_reconnect = should_allow_exhausted_shared_reconnect( @@ -391,6 +394,9 @@ pub(in crate::api) async fn m3u_api_stream_loaded( session_url.as_ref(), ); if connection_permission == UserConnectionPermission::Exhausted && !allow_exhausted_shared_reconnect { + if connection_admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } if extension == HLS_EXT { return hls_admission_failure_manifest_response( app_state, diff --git a/backend/app/src/api/endpoints/v1_api.rs b/backend/app/src/api/endpoints/v1_api.rs index 2b242879c..361ca4656 100644 --- a/backend/app/src/api/endpoints/v1_api.rs +++ b/backend/app/src/api/endpoints/v1_api.rs @@ -58,7 +58,7 @@ pub async fn create_status_check(app_state: &Arc) -> StatusCheck { }; let active_provider_connections = - app_state.active_provider.active_connections().await.map(|c| c.into_iter().collect::>()); + app_state.active_provider.active_connections().map(|c| c.into_iter().collect::>()); StatusCheck { status: "ok".to_string(), @@ -71,6 +71,7 @@ pub async fn create_status_check(app_state: &Arc) -> StatusCheck { active_provider_connections, active_user_streams, cache, + reentry_suppressed_total: app_state.active_users.reentry_suppressed_total(), } } async fn status(axum::extract::State(app_state): axum::extract::State>) -> axum::response::Response { diff --git a/backend/app/src/api/endpoints/v1_api_config.rs b/backend/app/src/api/endpoints/v1_api_config.rs index bdc79219d..a5d09b3ff 100644 --- a/backend/app/src/api/endpoints/v1_api_config.rs +++ b/backend/app/src/api/endpoints/v1_api_config.rs @@ -25,7 +25,8 @@ use shared::{ error::TuliproxError, model::{ permission::{Permission, PermissionSet}, - ApiProxyConfigDto, ConfigDto, InputFetchMethod, PlansConfigDto, SourcesConfigDto, XtreamLoginRequest, + ApiProxyConfigDto, ApiProxyServerInfoDto, ConfigDto, ConfigTargetDto, InputFetchMethod, PlansConfigDto, + SourcesConfigDto, XtreamLoginRequest, }, utils::{ parse_provider_scheme_url_parts, HEADER_CONFIG_API_PROXY_REVISION, HEADER_CONFIG_MAIN_REVISION, @@ -239,9 +240,22 @@ fn decode_permissions(app_state: &AppState, token: &str) -> Option ApiProxyServerInfoDto { + ApiProxyServerInfoDto { name: server.name.clone(), ..Default::default() } +} + fn filter_api_proxy_by_permissions(api_proxy: &mut ApiProxyConfigDto, permissions: PermissionSet) { if !permissions.contains(Permission::ConfigRead) { - api_proxy.server.clear(); + if permissions.contains(Permission::UserRead) || permissions.contains(Permission::UserWrite) { + api_proxy.server = api_proxy.server.iter().map(sanitize_server_names_only).collect(); + } else { + api_proxy.server.clear(); + } api_proxy.use_user_db = false; api_proxy.auth_error_status = ApiProxyConfigDto::default().auth_error_status; } @@ -250,6 +264,15 @@ fn filter_api_proxy_by_permissions(api_proxy: &mut ApiProxyConfigDto, permission } } +/// Reduces a target to the identity a user record may reference. +/// +/// User records reference a target by name; the id is kept so the UI can key +/// rows. Every other field (filter, output, mappings, ...) is target +/// configuration and must not leave the server without `SourceRead`. +fn sanitize_target_identity_only(target: &ConfigTargetDto) -> ConfigTargetDto { + ConfigTargetDto { id: target.id, name: target.name.clone(), ..Default::default() } +} + fn filter_app_config_by_permissions(app_config: &mut shared::model::AppConfigDto, permissions: Option) { // Unconditional: `config.yml` goes out in full to anyone with // `ConfigRead`, which included the Telegram bot token, the Pushover @@ -262,23 +285,30 @@ fn filter_app_config_by_permissions(app_config: &mut shared::model::AppConfigDto if let Some(permissions) = permissions { if !permissions.contains(Permission::ConfigRead) { app_config.config = ConfigDto::default(); - if let Some(api_proxy) = app_config.api_proxy.as_mut() { - api_proxy.server.clear(); - api_proxy.use_user_db = false; - api_proxy.auth_error_status = ApiProxyConfigDto::default().auth_error_status; - } } if !permissions.contains(Permission::SourceRead) { - app_config.sources = SourcesConfigDto::default(); app_config.mappings = None; app_config.templates = None; + let can_read_targets = permissions.contains(Permission::UserRead) + || permissions.contains(Permission::UserWrite) + || permissions.contains(Permission::PlaylistRead) + || permissions.contains(Permission::PlaylistWrite); + if can_read_targets { + app_config.sources.inputs.clear(); + app_config.sources.provider = None; + app_config.sources.templates = None; + for source in &mut app_config.sources.sources { + source.inputs.clear(); + source.targets = source.targets.iter().map(sanitize_target_identity_only).collect(); + } + } else { + app_config.sources = SourcesConfigDto::default(); + } } if let Some(api_proxy) = app_config.api_proxy.as_mut() { - if !permissions.contains(Permission::UserRead) { - api_proxy.user.clear(); - } + filter_api_proxy_by_permissions(api_proxy, permissions); } } } @@ -475,7 +505,7 @@ async fn save_config_sources( .await; } - app_state.active_provider.update_config(&app_state.app_config).await; + app_state.active_provider.update_config(&app_state.app_config); let updated_revision = match read_file_revision(&sources_file_path).await { Ok(revision) => revision, Err(err) => { @@ -653,7 +683,17 @@ async fn config( let Some(permissions) = decode_permissions(&app_state, &token) else { return axum::http::StatusCode::UNAUTHORIZED.into_response(); }; - if !has_any_permission(permissions, &[Permission::ConfigRead, Permission::SourceRead, Permission::UserRead]) { + if !has_any_permission( + permissions, + &[ + Permission::ConfigRead, + Permission::SourceRead, + Permission::UserRead, + Permission::UserWrite, + Permission::PlaylistRead, + Permission::PlaylistWrite, + ], + ) { return axum::http::StatusCode::FORBIDDEN.into_response(); } @@ -667,7 +707,7 @@ async fn get_config_api_proxy_config( let Some(permissions) = decode_permissions(&app_state, &token) else { return axum::http::StatusCode::UNAUTHORIZED.into_response(); }; - if !has_any_permission(permissions, &[Permission::ConfigRead, Permission::UserRead]) { + if !has_any_permission(permissions, &[Permission::ConfigRead, Permission::UserRead, Permission::UserWrite]) { return axum::http::StatusCode::FORBIDDEN.into_response(); } @@ -808,24 +848,47 @@ fn build_xtream_login_input_source( }) } -async fn get_config_plans( - axum::extract::State(app_state): axum::extract::State>, -) -> impl IntoResponse + Send { +fn get_config_plans_dto(app_state: &Arc) -> Result { let plans_path = { let paths = app_state.app_config.paths.load(); plans_file_path(paths.api_proxy_file_path.as_str()) }; let plans_path_str = plans_path.to_string_lossy().to_string(); match read_plans_file(&plans_path_str, true) { - Ok(Some(dto)) => axum::response::Json(dto).into_response(), - Ok(None) => axum::response::Json(PlansConfigDto::default()).into_response(), + Ok(Some(dto)) => Ok(dto), + Ok(None) => Ok(PlansConfigDto::default()), Err(err) => { error!("Failed to read plans config: {err}"); - internal_server_error!() + Err(err) } } } +async fn get_config_plans( + AuthBearer(token): AuthBearer, + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { + let Some(permissions) = decode_permissions(&app_state, &token) else { + return axum::http::StatusCode::UNAUTHORIZED.into_response(); + }; + if !has_any_permission(permissions, &[Permission::ConfigRead, Permission::UserRead, Permission::UserWrite]) { + return axum::http::StatusCode::FORBIDDEN.into_response(); + } + match get_config_plans_dto(&app_state) { + Ok(dto) => axum::response::Json(dto).into_response(), + Err(_) => internal_server_error!(), + } +} + +async fn get_config_plans_unprotected( + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { + match get_config_plans_dto(&app_state) { + Ok(dto) => axum::response::Json(dto).into_response(), + Err(_) => internal_server_error!(), + } +} + async fn save_config_plans( axum::extract::State(app_state): axum::extract::State>, axum::extract::Json(mut req_plans): axum::extract::Json, @@ -866,15 +929,13 @@ pub fn v1_api_config_register(router: Router>) -> axum::Router) -> Router> { let base_read = Router::new() .route("/config", axum::routing::get(config)) - .route("/config/apiproxy", axum::routing::get(get_config_api_proxy_config)); - - let config_read = Router::new() - .route("/config/plans", axum::routing::get(get_config_plans)) - .layer(permission_layer!(app_state, Permission::ConfigRead)); + .route("/config/apiproxy", axum::routing::get(get_config_api_proxy_config)) + .route("/config/plans", axum::routing::get(get_config_plans)); // 2. Source Domain (Read & Write) let source_read = Router::new() @@ -893,7 +954,7 @@ pub fn v1_api_config_register_with_permissions(app_state: &Arc) -> Rou .route("/config/plans", axum::routing::put(save_config_plans)) .layer(permission_layer!(app_state, Permission::ConfigWrite)); - Router::new().merge(base_read).merge(config_read).merge(source_read).merge(source_write).merge(config_write) + Router::new().merge(base_read).merge(source_read).merge(source_write).merge(config_write) } /// Request body for `POST /config/messaging/test`. @@ -1060,18 +1121,121 @@ mod tests { } #[test] - fn filter_api_proxy_keeps_user_section_only_with_user_read() { + fn filter_api_proxy_keeps_user_and_sanitized_server_section_with_user_read() { let permissions: PermissionSet = Permission::UserRead.into(); let mut api_proxy = make_test_api_proxy(); filter_api_proxy_by_permissions(&mut api_proxy, permissions); - assert!(api_proxy.server.is_empty()); + assert_eq!(api_proxy.server.len(), 1); + assert_eq!(api_proxy.server[0].name, "main"); + assert!(api_proxy.server[0].protocol.is_empty()); + assert!(api_proxy.server[0].host.is_empty()); + assert!(api_proxy.server[0].port.is_none()); + assert!(api_proxy.server[0].timezone.is_empty()); + assert!(api_proxy.server[0].message.is_empty()); + assert!(api_proxy.server[0].path.is_none()); assert_eq!(api_proxy.user.len(), 1); assert!(!api_proxy.use_user_db); assert_eq!(api_proxy.auth_error_status, 403); } + #[test] + fn filter_api_proxy_keeps_sanitized_server_section_with_user_write() { + let permissions: PermissionSet = Permission::UserWrite.into(); + let mut api_proxy = make_test_api_proxy(); + + filter_api_proxy_by_permissions(&mut api_proxy, permissions); + + assert_eq!(api_proxy.server.len(), 1); + assert_eq!(api_proxy.server[0].name, "main"); + assert!(api_proxy.server[0].host.is_empty()); + assert!(api_proxy.user.is_empty()); + } + + #[test] + fn filter_api_proxy_clears_all_for_unrelated_permission() { + let permissions: PermissionSet = Permission::SystemRead.into(); + let mut api_proxy = make_test_api_proxy(); + + filter_api_proxy_by_permissions(&mut api_proxy, permissions); + + assert!(api_proxy.server.is_empty()); + assert!(api_proxy.user.is_empty()); + assert!(!api_proxy.use_user_db); + assert_eq!(api_proxy.auth_error_status, 403); + } + + #[test] + fn filter_app_config_keeps_targets_and_clears_sources_for_user_read_without_source_read() { + let permissions: PermissionSet = Permission::UserRead.into(); + let mut app_config = AppConfigDto { + config: ConfigDto { storage_dir: Some(String::from("storage")), ..ConfigDto::default() }, + sources: SourcesConfigDto { + inputs: vec![shared::model::ConfigInputDto { + name: Arc::from("secret-provider"), + url: String::from("http://secret-provider/get.php?username=foo&password=bar"), + ..Default::default() + }], + sources: vec![shared::model::ConfigSourceDto { + inputs: vec![Arc::from("secret-provider")], + targets: vec![shared::model::ConfigTargetDto { + id: 1, + enabled: false, + name: String::from("Default"), + output: vec![shared::model::TargetOutputDto::M3u(shared::model::M3uTargetOutputDto { + filename: Some(String::from("secret.m3u")), + ..Default::default() + })], + filter: shared::model::ConfigTargetFilterDto { + processing: Some(String::from("group ~ 'secret'")), + ..Default::default() + }, + watch: Some(vec![String::from("watch-expr")]), + mapping: Some(vec![String::from("mapping-name")]), + ..Default::default() + }], + }], + provider: Some(vec![]), + templates: Some(vec![]), + }, + mappings: None, + templates: Some(TemplateDefinitionDto::default()), + api_proxy: Some(make_test_api_proxy()), + }; + + filter_app_config_by_permissions(&mut app_config, Some(permissions)); + + assert_eq!(app_config.config.storage_dir, None); + assert!(app_config.sources.inputs.is_empty()); + assert!(app_config.sources.provider.is_none()); + assert!(app_config.sources.templates.is_none()); + assert!(app_config.mappings.is_none()); + assert!(app_config.templates.is_none()); + assert_eq!(app_config.sources.sources.len(), 1); + assert!(app_config.sources.sources[0].inputs.is_empty()); + assert_eq!(app_config.sources.sources[0].targets.len(), 1); + + let target = &app_config.sources.sources[0].targets[0]; + assert_eq!(target.name, "Default"); + assert_eq!(target.id, 1); + assert_eq!(target.enabled, shared::model::ConfigTargetDto::default().enabled); + assert!(target.output.is_empty()); + assert!(target.filter.is_empty()); + assert!(target.options.is_none()); + assert!(target.sort.is_none()); + assert!(target.rename.is_none()); + assert!(target.mapping.is_none()); + assert!(target.favourites.is_none()); + assert!(target.watch.is_none()); + + let api_proxy = app_config.api_proxy.expect("api proxy should remain present"); + assert_eq!(api_proxy.server.len(), 1); + assert_eq!(api_proxy.server[0].name, "main"); + assert!(api_proxy.server[0].host.is_empty()); + assert_eq!(api_proxy.user.len(), 1); + } + #[test] fn build_xtream_login_input_source_preserves_provider_context_for_provider_urls() { let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { diff --git a/backend/app/src/api/endpoints/v1_api_playlist.rs b/backend/app/src/api/endpoints/v1_api_playlist.rs index d16865dbb..403170063 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist.rs @@ -68,6 +68,8 @@ use std::{path::Path, str::FromStr, sync::Arc}; use tokio_stream::StreamExt; use url::Url; +const PLAYLIST_UPDATE_STATUS_READ_CONCURRENCY: usize = 8; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ManualUpdateEnqueueError { Busy, @@ -635,24 +637,27 @@ async fn playlist_update_status( ) -> Result, axum::http::StatusCode> { let storage_dir = app_state.app_config.config.load().storage_dir.clone(); let inputs = app_state.app_config.sources.load().inputs.clone(); - let status_reads = futures::stream::iter(inputs.clone().into_iter().map(|input| { + let status_reads = futures::stream::iter(inputs.into_iter().map(|input| { let storage_dir = storage_dir.clone(); let input_name = input.name.clone(); async move { let storage_path = crate::processing::input_cache::resolve_input_storage_path(&storage_dir, &input_name).await; - read_playlist_update_input_status(move || crate::processing::input_cache::load_input_status(&storage_path)) - .await + let input_status = read_playlist_update_input_status(move || { + crate::processing::input_cache::load_input_status(&storage_path) + }) + .await?; + Ok::<_, axum::http::StatusCode>((input, input_status)) } })); let status_reads = { use futures::StreamExt as _; - status_reads.buffered(8) + status_reads.buffered(PLAYLIST_UPDATE_STATUS_READ_CONCURRENCY) }; let input_statuses: Vec<_> = futures::StreamExt::collect(status_reads).await; let input_statuses = input_statuses.into_iter().collect::, _>>()?; - let mut statuses = Vec::with_capacity(inputs.len()); - for (input, input_status) in inputs.into_iter().zip(input_statuses) { + let mut statuses = Vec::with_capacity(input_statuses.len()); + for (input, input_status) in input_statuses { let cluster_based = input.input_type.is_xtream() || input.input_type.is_stalker(); let last_update = input_status.clusters.values().map(|cluster| cluster.timestamp).filter(|timestamp| *timestamp > 0).max(); @@ -2586,7 +2591,7 @@ mod tests { let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); let history_config = Some(StreamHistoryConfig::default()); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let config = app_cfg.config.load(); diff --git a/backend/app/src/api/endpoints/v1_api_user.rs b/backend/app/src/api/endpoints/v1_api_user.rs index 2419b449a..9521f8366 100644 --- a/backend/app/src/api/endpoints/v1_api_user.rs +++ b/backend/app/src/api/endpoints/v1_api_user.rs @@ -272,7 +272,7 @@ async fn terminate_user_session( axum::extract::Path((username, session_token)): axum::extract::Path<(String, String)>, ) -> impl axum::response::IntoResponse { app_state.active_users.terminate_session(&username, &session_token).await; - app_state.active_provider.clear_provider_reservation(&session_token).await; + app_state.active_provider.clear_provider_reservation(&session_token); (axum::http::StatusCode::NO_CONTENT).into_response() } diff --git a/backend/app/src/api/endpoints/websocket_api.rs b/backend/app/src/api/endpoints/websocket_api.rs index fd9873cfd..9515972e8 100644 --- a/backend/app/src/api/endpoints/websocket_api.rs +++ b/backend/app/src/api/endpoints/websocket_api.rs @@ -285,7 +285,7 @@ async fn handle_protocol_message( None } Ok(ProtocolMessage::ActiveProviderCountRequest(auth_token)) => { - handle_active_provider_count_request(auth_token, mem, app_state, auth_required, verifier).await + Some(handle_active_provider_count_request(auth_token, mem, app_state, auth_required, verifier)) } Ok(_) => { trace!("Unexpected protocol message after handshake"); @@ -318,32 +318,32 @@ fn handle_stream_meter_unsubscribe(mem: &mut ProtocolHandlerMemory, app_state: & } } -async fn handle_active_provider_count_request( +fn handle_active_provider_count_request( auth_token: String, mem: &mut ProtocolHandlerMemory, app_state: &Arc, auth_required: bool, verifier: Option<&TokenVerifier>, -) -> Option { +) -> ProtocolMessage { if auth_required { let Some(verifier) = verifier else { - return Some(ProtocolMessage::Unauthorized); + return ProtocolMessage::Unauthorized; }; let Some(token_data) = verifier.verify(&auth_token) else { - return Some(ProtocolMessage::Unauthorized); + return ProtocolMessage::Unauthorized; }; if token_data.claims.permissions.contains(Permission::SystemRead) && set_websocket_auth(mem, auth_token, &token_data.claims) { - let connections = app_state.active_provider.get_provider_connections_count().await; - Some(ProtocolMessage::ActiveProviderCountResponse(connections)) + let connections = app_state.active_provider.get_provider_connections_count(); + ProtocolMessage::ActiveProviderCountResponse(connections) } else { - Some(ProtocolMessage::Unauthorized) + ProtocolMessage::Unauthorized } } else { set_no_auth_websocket_identity(mem, Some(TOKEN_NO_AUTH.to_string())); - let connections = app_state.active_provider.get_provider_connections_count().await; - Some(ProtocolMessage::ActiveProviderCountResponse(connections)) + let connections = app_state.active_provider.get_provider_connections_count(); + ProtocolMessage::ActiveProviderCountResponse(connections) } } diff --git a/backend/app/src/api/endpoints/xtream_api/mod.rs b/backend/app/src/api/endpoints/xtream_api/mod.rs index 6ccaedf5f..b5a50b96e 100644 --- a/backend/app/src/api/endpoints/xtream_api/mod.rs +++ b/backend/app/src/api/endpoints/xtream_api/mod.rs @@ -10,8 +10,8 @@ use crate::{ empty_json_response_as_array, empty_json_response_as_object, force_provider_stream_response, get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, internal_server_error, is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, local_stream_response, - redirect, redirect_response, resolve_initial_stalker_playback_url, resource_response, - separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, + redirect, redirect_response, reentry_suppressed_response, resolve_initial_stalker_playback_url, + resource_response, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, try_option_bad_request, try_result_bad_request, try_unwrap_body, RedirectParams, ResourceFetchPolicy, }, endpoints::{ @@ -408,6 +408,9 @@ async fn xtream_player_api_stream( false, ) .await; + if admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } return local_stream_response( fingerprint, app_state, @@ -416,8 +419,8 @@ async fn xtream_player_api_stream( &input, &target, &user, - admission.permission, - admission.kind.unwrap_or(crate::api::model::ConnectionKind::Normal), + admission.permission(), + admission.kind().unwrap_or(crate::api::model::ConnectionKind::Normal), Some(playback_session_token.as_str()), Some(request_class), true, @@ -470,7 +473,7 @@ async fn xtream_player_api_stream( let session_key = if let Some(resolved) = m3u_timeshift.as_ref() { create_m3u_catchup_session_key(fingerprint, &user.username, virtual_id.get(), &resolved.discriminator) } else if item_type == PlaylistItemType::Catchup { - create_catchup_session_key(fingerprint, &user.username, virtual_id.get()) + create_catchup_session_key(fingerprint, &user.username, virtual_id.get(), stream_req.action_path) } else { create_playback_session_fingerprint( fingerprint, @@ -515,7 +518,7 @@ async fn xtream_player_api_stream( ); } - if app_state.active_provider.is_over_limit(&session.provider).await { + if app_state.active_provider.is_over_limit(&session.provider) { let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); if playback_ext == HLS_EXT { return hls_admission_failure_manifest_response( @@ -581,9 +584,9 @@ async fn xtream_player_api_stream( false, ) .await; - let connection_permission = connection_admission.permission; + let connection_permission = connection_admission.permission(); let connection_kind = connection_admission - .kind + .kind() .or(user_session.as_ref().and_then(|session| session.connection_kind)) .unwrap_or(crate::api::model::ConnectionKind::Normal); let allow_exhausted_shared_reconnect = should_allow_exhausted_shared_reconnect( @@ -593,6 +596,9 @@ async fn xtream_player_api_stream( session_url.as_ref(), ); if connection_permission == UserConnectionPermission::Exhausted && !allow_exhausted_shared_reconnect { + if connection_admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); if playback_ext == HLS_EXT { return hls_admission_failure_manifest_response( @@ -671,7 +677,7 @@ async fn xtream_player_api_stream( &input, req_headers, connection_permission, - connection_admission.kind, + connection_admission.kind(), &original_hls_entry_path, ) .await diff --git a/backend/app/src/api/main_api.rs b/backend/app/src/api/main_api.rs index 74004b209..bf973ce9f 100644 --- a/backend/app/src/api/main_api.rs +++ b/backend/app/src/api/main_api.rs @@ -48,6 +48,7 @@ use axum::{ use dashmap::DashSet; use log::{debug, error, info, warn}; use shared::{ + defaults::default_cleanup_queue_capacity, error::TuliproxError, model::{PlaylistUpdateState, PlaylistUpdateSummary, ServerLifecycleEvent}, utils::{concat_path_leading_slash, sanitize_sensitive_info}, @@ -280,11 +281,16 @@ fn get_web_dir_path(web_ui_enabled: bool, web_root: &str) -> Result Healthcheck { + // `status` stays `ok` for as long as the server answers requests: the + // container healthcheck restarts on a non-ok body, and a wedged runtime is + // reported by the watchdog thread instead of by killing the process. + // Liveness is exposed separately in `runtime`. Healthcheck { status: "ok".to_string(), version: VERSION.to_string(), build_time: get_build_time(), server_time: get_server_time(), + runtime: tuliprox_core::utils::runtime_liveness::health_snapshot(), } } @@ -301,7 +307,7 @@ async fn ready( use crate::model::readiness::build_provider_slots; use shared::model::provider_saturation::is_exhausted; let sources = app_state.app_config.sources.load(); - let Some(connections) = app_state.active_provider.active_connections().await else { + let Some(connections) = app_state.active_provider.active_connections() else { // No live connections yet: either the lineups are still warming up, or // there is no enabled input that could ever carry one. let status = if sources.inputs.iter().any(|input| input.enabled) { "initializing" } else { "exhausted" }; @@ -402,17 +408,23 @@ async fn create_shared_data( config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()), &rewrite_secret, )); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); active_users.start_adaptive_expiry_worker(); let history_config = config.reverse_proxy.as_ref().and_then(|r| r.stream_history.as_ref()); - let connection_manager = Arc::new(ConnectionManager::new( + let cleanup_capacity = config + .reverse_proxy + .as_ref() + .and_then(|r| r.stream.as_ref()) + .map_or_else(default_cleanup_queue_capacity, |stream| stream.cleanup_queue_capacity); + let connection_manager = Arc::new(ConnectionManager::new_with_capacity( &active_users, &active_provider, &shared_stream_manager, &event_manager, history_config, + cleanup_capacity, )); let client = create_http_client(app_config)?; @@ -1176,7 +1188,7 @@ mod tests { let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let config = app_cfg.config.load(); let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); @@ -1249,7 +1261,6 @@ mod tests { state .active_provider .acquire_connection(&input.into(), &addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("connection allocation") } diff --git a/backend/app/src/api/model/app_state.rs b/backend/app/src/api/model/app_state.rs index b8f653598..25e0904e5 100644 --- a/backend/app/src/api/model/app_state.rs +++ b/backend/app/src/api/model/app_state.rs @@ -151,7 +151,7 @@ pub async fn update_app_state_sources( Arc::new(targets) }; app_state.forced_targets.store(targets); - let updates = app_state.set_sources(sources).await?; + let updates = app_state.set_sources(sources)?; update_target_caches(app_state, updates.targets.as_ref()).await; restart_services(app_state, &updates); Ok(()) @@ -497,13 +497,25 @@ pub(crate) fn create_test_app_state(config: Config) -> Arc { let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(&app_config, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let loaded_config = app_config.config.load(); let active_users = Arc::new(ActiveUserManager::new(&loaded_config, &geoip, &event_manager)); - let connection_manager = - Arc::new(ConnectionManager::new(&active_users, &active_provider, &shared_stream_manager, &event_manager, None)); + let cleanup_capacity = loaded_config + .reverse_proxy + .as_ref() + .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) + .map_or_else(shared::defaults::default_cleanup_queue_capacity, |stream| stream.cleanup_queue_capacity); + drop(loaded_config); + let connection_manager = Arc::new(ConnectionManager::new_with_capacity( + &active_users, + &active_provider, + &shared_stream_manager, + &event_manager, + None, + cleanup_capacity, + )); let tokens = CancelTokens::default(); let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); let (manual_update_sender, _) = mpsc::channel::(1); @@ -566,7 +578,7 @@ impl AppState { self.active_users.update_config(&config); self.app_config.set_config(config)?; reload_logger(config_log_level.as_deref()); - self.active_provider.update_config(&self.app_config).await; + self.active_provider.update_config(&self.app_config); self.hls_proxy.update_config(&self.app_config).await; self.update_config().await?; @@ -618,10 +630,7 @@ impl AppState { Ok(()) } - pub(in crate::api::model) async fn set_sources( - &self, - sources: SourcesConfig, - ) -> Result { + pub(in crate::api::model) fn set_sources(&self, sources: SourcesConfig) -> Result { let changes = self.detect_changes_for_sources(&sources); // Carry over DNS caches from old providers so resolved IPs survive hot-reloads // without waiting for the background resolver or the persisted-file seed. @@ -640,7 +649,7 @@ impl AppState { } } self.app_config.set_sources(sources)?; - self.active_provider.update_config(&self.app_config).await; + self.active_provider.update_config(&self.app_config); shared::model::REGEX_CACHE.sweep(); Ok(changes) diff --git a/backend/app/src/api/model/streams/active_client_stream.rs b/backend/app/src/api/model/streams/active_client_stream.rs index d7fc36041..7df5f8682 100644 --- a/backend/app/src/api/model/streams/active_client_stream.rs +++ b/backend/app/src/api/model/streams/active_client_stream.rs @@ -1,11 +1,13 @@ use crate::{ api::{ model::{ - connection_manager::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET}, - create_provider_stream, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, CleanupEvent, - ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream, PendingProviderWakeSource, - ProviderHandle, ProviderStreamFactoryOptions, StreamDetails, StreamError, StreamMeterHandle, - TimedClientStream, TransportStreamBuffer, + connection_manager::{ + PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET, PROVIDER_END_PREEMPTED, + }, + open_provider_stream_with_lifecycle, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, + CleanupEvent, ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream, + PendingProviderWakeSource, ProviderStreamFactoryOptions, ProviderStreamOpenLifecycle, StreamDetails, + StreamError, StreamMeterHandle, TimedClientStream, TransportStreamBuffer, DIRECT_BODY_IDLE_TIMEOUT_SECS, }, panel_api::{can_provision_on_exhausted, find_input_by_provider_name, run_panel_api_provisioning_probe}, }, @@ -32,9 +34,8 @@ use std::{ }; use tokio::sync::Notify; use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; -use tuliprox_session::stream_options::get_stream_options; +use tuliprox_session::{stream_options::get_stream_options, ConnectionRejectionReason}; -const DIRECT_BODY_IDLE_TIMEOUT_SECS: u64 = 90; const BODY_IDLE_TIMEOUT_ERROR_CLASS: &str = "body_idle_timeout"; const DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS: u64 = 1; @@ -42,39 +43,58 @@ const DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS: u64 = 1; /// Stored as `u8` in an `AtomicU8` for lock-free access inside `poll_next`. /// Lower numeric values correspond to a live or custom stream; `GracePending` /// (255) is a transient sentinel that parks the poll until the grace task resolves. +/// +/// Discriminants are implicit (declaration order) except for the `255` sentinel. +/// The byte mapping lives once, in [`StreamMode::try_from`], and is pinned by +/// `test_stream_mode_byte_values`. #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum StreamMode { /// Forward bytes directly from the upstream provider. - Inner = 0, + Inner, /// Show the "user connections exhausted" custom video. - UserExhausted = 1, + UserExhausted, /// Show the "provider connections exhausted" custom video. - ProviderExhausted = 2, + ProviderExhausted, /// Show the "channel unavailable" custom video. - ChannelUnavailable = 3, + ChannelUnavailable, /// Show the provisioning/placeholder custom video while probing for capacity. - Provisioning = 4, + Provisioning, /// Show the "low-priority preempted" custom video. - LowPriorityPreempted = 5, + LowPriorityPreempted, + /// A recently evicted playback retried while this stream held a grace slot and + /// every remaining eviction candidate is reentry-protected. The body must end + /// without painting a user-visible "connections exhausted" error video. + ReentrySuppressed, /// Transient: grace-period check is still in progress; `poll_next` must park. GracePending = 255, } -impl StreamMode { - fn from_u8(v: u8) -> Self { - match v { - 0 => Self::Inner, - 1 => Self::UserExhausted, - 2 => Self::ProviderExhausted, - 3 => Self::ChannelUnavailable, - 4 => Self::Provisioning, - 5 => Self::LowPriorityPreempted, - _ => Self::GracePending, +impl TryFrom for StreamMode { + type Error = u8; + + /// Decodes the atomic mode flag. Returns `Err(value)` for a byte that was never a + /// valid [`StreamMode`]; callers must fail safe instead of inventing a live state + /// such as `GracePending`. + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Inner), + 1 => Ok(Self::UserExhausted), + 2 => Ok(Self::ProviderExhausted), + 3 => Ok(Self::ChannelUnavailable), + 4 => Ok(Self::Provisioning), + 5 => Ok(Self::LowPriorityPreempted), + 6 => Ok(Self::ReentrySuppressed), + 255 => Ok(Self::GracePending), + other => Err(other), } } } +/// Publishes a new mode to the lock-free flag with release ordering. All writers go +/// through this so the `as u8` cast and the memory ordering live in one place. +fn store_stream_mode(flag: &AtomicU8, mode: StreamMode) { flag.store(mode as u8, Ordering::Release); } + /// Holds the optional custom video buffers for each error/placeholder scenario. /// Using named fields avoids the positional-indexing confusion of a 4-tuple. struct CustomVideoBuffers { @@ -143,6 +163,7 @@ struct GracePeriodParams { // Whether the session is `socket_bound`. Used to construct the correct // `EvictionReentryGuard`. socket_bound: bool, + shared_subscriber_id: Option, } enum DeferredProviderOpenOutcome { @@ -209,7 +230,7 @@ impl DirectBodyIdleTimeout { struct ActiveClientStreamState { inner: Option, send_custom_stream_flag: Option>, - provider_handle: Option, + provider_handle: Option, deferred_provider_open: Option, timed_stream_context: Option, preempt_cancelled: Option>>, @@ -241,6 +262,16 @@ struct ActiveClientStreamState { provider_http_status: Option, /// Count of successful provider reconnections during this session (grace period / deferred open). provider_reconnect_count: AtomicU8, + /// Playback lease owner (session token) whose provider slot is confirmed once real + /// media bytes reach the client. `None` when this stream has no provider lease. + lease_owner: Option>, + /// Guards against emitting the confirmation more than once per stream. + lease_confirmed: bool, + lease_request_id: Option, + /// Guaranteed final cleanup for the registered request claim. The body owns this and + /// finishes it exactly once with the real provider outcome, so a full cleanup queue can + /// never leak the user request or provider lease. + request_cleanup: Option, } impl ActiveClientStreamState { @@ -269,6 +300,7 @@ impl ActiveClientStreamState { ctx.duration_secs, self.fingerprint.addr, ctx.virtual_id, + self.stream_uid, ) .boxed() } else { @@ -276,15 +308,28 @@ impl ActiveClientStreamState { } } - fn custom_video_type_for_mode(mode: StreamMode) -> CustomVideoStreamType { + /// Maps a mode to the custom video it serves, if any. `ReentrySuppressed` and the + /// transparent forwarding modes deliberately have no custom video: a suppressed + /// reentry must never surface a user-visible error clip. + fn custom_video_type_for_mode(mode: StreamMode) -> Option { match mode { - StreamMode::UserExhausted => CustomVideoStreamType::UserConnectionsExhausted, - StreamMode::ProviderExhausted => CustomVideoStreamType::ProviderConnectionsExhausted, - StreamMode::Provisioning => CustomVideoStreamType::Provisioning, - StreamMode::LowPriorityPreempted => CustomVideoStreamType::LowPriorityPreempted, - StreamMode::ChannelUnavailable | StreamMode::Inner | StreamMode::GracePending => { - CustomVideoStreamType::ChannelUnavailable - } + StreamMode::UserExhausted => Some(CustomVideoStreamType::UserConnectionsExhausted), + StreamMode::ProviderExhausted => Some(CustomVideoStreamType::ProviderConnectionsExhausted), + StreamMode::Provisioning => Some(CustomVideoStreamType::Provisioning), + StreamMode::LowPriorityPreempted => Some(CustomVideoStreamType::LowPriorityPreempted), + StreamMode::ChannelUnavailable => Some(CustomVideoStreamType::ChannelUnavailable), + StreamMode::Inner | StreamMode::GracePending | StreamMode::ReentrySuppressed => None, + } + } + + /// Drops the provider handle at most once (releasing the provider slot + /// synchronously) and detaches the provider body stream. The stream claim itself + /// is released separately through [`Self::release_user_stream`]. + fn release_provider_handle_and_detach_body(&mut self) { + self.inner = None; + if !self.provider_handle_released { + self.provider_handle_released = true; + drop(self.provider_handle.take()); } } @@ -293,44 +338,46 @@ impl ActiveClientStreamState { return; } self.user_stream_released = true; - self.connection_manager.send_cleanup(CleanupEvent::ReleaseStream { - addr: self.fingerprint.addr, - stream_uid: self.stream_uid, - provider_end_reason: self.provider_end_reason.load(Ordering::Relaxed), - reconnect_count: self.provider_reconnect_count.load(Ordering::Relaxed), - provider_error_class: self.provider_error_class, - provider_http_status: self.provider_http_status, + if let Some(cleanup) = self.request_cleanup.as_mut() { + cleanup.finish( + self.lease_request_id, + self.provider_end_reason.load(Ordering::Relaxed), + self.provider_reconnect_count.load(Ordering::Relaxed), + self.provider_error_class, + self.provider_http_status, + ); + } + } + + /// Confirms the playback lease exactly once, when the first real provider media + /// bytes reach the client. Only a confirmed lease may reserve provider capacity + /// against other playbacks, so an abandoned or custom-video-only start does not + /// block unrelated clients behind the same reverse proxy. + fn confirm_lease_once(&mut self) { + if self.lease_confirmed { + return; + } + let Some(owner) = self.lease_owner.take() else { + return; + }; + self.lease_confirmed = true; + let request_id = self.lease_request_id.or_else(|| { + self.provider_handle.as_ref().and_then(|managed| managed.handle()).and_then(|h| h.playback_request_id) }); + // Confirm synchronously through the broker: a first-byte confirmation is a + // mandatory capacity transition and must not be dropped by queue pressure. + let provider_manager = &self.connection_manager.provider_manager; + if let Some(request_id) = request_id { + provider_manager.confirm_identified_playback_activity(&owner, request_id); + } else { + provider_manager.confirm_playback_activity(&owner); + } } fn release_stream_and_provider_handle_once(&mut self) { self.stop_grace_task(); - let addr = self.fingerprint.addr; - let handle = self.provider_handle.take(); - // `provider_handle_released` mirrors `user_stream_released` for the provider slot. - // When preemption already released the handle, `provider_handle` is None and the - // flag is true; sending None is a no-op, but the explicit guard keeps the invariant - // visible and safe against future call-site additions. - let handle_for_cleanup = if self.provider_handle_released { None } else { handle }; - if self.user_stream_released { - if !self.provider_handle_released { - self.provider_handle_released = true; - self.connection_manager - .send_cleanup(CleanupEvent::ReleaseProviderHandle { handle: handle_for_cleanup }); - } - } else { - self.user_stream_released = true; - self.provider_handle_released = true; - self.connection_manager.send_cleanup(CleanupEvent::ReleaseStreamAndProviderHandle { - addr, - stream_uid: self.stream_uid, - handle: handle_for_cleanup, - provider_end_reason: self.provider_end_reason.load(Ordering::Relaxed), - reconnect_count: self.provider_reconnect_count.load(Ordering::Relaxed), - provider_error_class: self.provider_error_class, - provider_http_status: self.provider_http_status, - }); - } + self.release_provider_handle_and_detach_body(); + self.release_user_stream(); } fn stop_direct_body_idle_timeout(&mut self) { @@ -388,22 +435,33 @@ impl ActiveClientStreamState { self.provider_stopped = true; self.preempt_cancelled = None; self.stop_grace_task(); + self.provider_end_reason.store(PROVIDER_END_PREEMPTED, Ordering::Relaxed); + self.provider_error_class = Some("preempted"); + + let is_superseded = self + .provider_handle + .as_ref() + .and_then(|m| m.handle()) + .is_some_and(|h| h.get_close_reason() == tuliprox_core::model::ProviderCloseReason::Superseded); let mut serve_preempted_custom = false; if self.provider_handle.is_some() { - let handle = self.provider_handle.take(); + let managed = self.provider_handle.take(); self.provider_handle_released = true; - if self.custom_video.low_priority_preempted.is_some() { + // Synchronous provider-slot release; the custom-video detail update is a + // separate, best-effort UI effect. + drop(managed); + if !is_superseded && self.custom_video.low_priority_preempted.is_some() { serve_preempted_custom = true; if let Some(flag) = &self.send_custom_stream_flag { - flag.store(StreamMode::LowPriorityPreempted as u8, Ordering::Release); + store_stream_mode(flag, StreamMode::LowPriorityPreempted); } else { // Fallback: create_active_client_stream usually initializes this via stream_grace_period. self.send_custom_stream_flag = Some(Arc::new(AtomicU8::new(StreamMode::LowPriorityPreempted as u8))); } } else if let Some(flag) = &self.send_custom_stream_flag { - flag.store(StreamMode::Inner as u8, Ordering::Release); + store_stream_mode(flag, StreamMode::Inner); } if let Some(waker) = &self.waker { @@ -422,12 +480,12 @@ impl ActiveClientStreamState { if serve_preempted_custom { self.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProvider { addr, + stream_uid: self.stream_uid, video_type: CustomVideoStreamType::LowPriorityPreempted, - handle, + handle: None, }); } else { self.release_user_stream(); - self.connection_manager.send_cleanup(CleanupEvent::ReleaseProviderHandle { handle }); } } serve_preempted_custom @@ -439,8 +497,9 @@ impl ActiveClientStreamState { self.stop_grace_task(); if self.provider_handle.is_some() { - let handle = self.provider_handle.take(); - self.provider_handle_released = true; + // Synchronous provider-slot release; the custom-video detail update is a + // separate, best-effort UI effect. + self.release_provider_handle_and_detach_body(); if mode == StreamMode::ChannelUnavailable { if let Some(flag) = &self.send_custom_stream_flag { @@ -458,31 +517,45 @@ impl ActiveClientStreamState { } let addr = self.fingerprint.addr; - // Drop the provider stream immediately instead of replacing with an - // allocated empty stream β€” avoids a heap allocation on every mode switch. - self.inner = None; - - let video_type = Self::custom_video_type_for_mode(mode); let reason = match mode { StreamMode::ChannelUnavailable => "unavailable provider channel", StreamMode::UserExhausted => "user grace period exhaustion", StreamMode::ProviderExhausted => "provider grace period exhaustion", StreamMode::Provisioning => "provider grace period provisioning", StreamMode::LowPriorityPreempted => "low-priority preemption", + StreamMode::ReentrySuppressed => "reentry suppression", StreamMode::Inner | StreamMode::GracePending => "stream mode transition", }; debug_if_enabled!( "Provider stream stopped due to {reason} for {}", sanitize_sensitive_info(&addr.to_string()) ); - self.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProvider { - addr, - video_type, - handle, - }); + if let Some(video_type) = Self::custom_video_type_for_mode(mode) { + self.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProvider { + addr, + stream_uid: self.stream_uid, + video_type, + handle: None, + }); + } else { + // No custom video for this mode: release the request without publishing + // a detail update so no error clip is shown. + self.release_user_stream(); + } } } + /// Ends the response body without switching to a custom error video. Used for a + /// suppressed reentry retry, where a visible "connections exhausted" video would be + /// misleading: the request was declined by the reentry guard, not by a real limit. + fn terminate_quietly(&mut self) { + self.provider_stopped = true; + self.preempt_cancelled = None; + self.stop_grace_task(); + self.release_provider_handle_and_detach_body(); + self.release_user_stream(); + } + fn reset_custom_video_timeout(&mut self) { self.custom_video_timeout_mode = None; self.custom_video_timeout_sleep = None; @@ -531,6 +604,7 @@ fn wrap_timed_client_stream_if_needed( stream: BoxedProviderStream, addr: SocketAddr, virtual_id: VirtualId, + stream_uid: Option, ) -> BoxedProviderStream { let config = app_state.app_config.config.load(); match config.sleep_timer_mins { @@ -545,6 +619,7 @@ fn wrap_timed_client_stream_if_needed( secs, addr, virtual_id, + stream_uid, ) .boxed() } else { @@ -639,9 +714,19 @@ impl Stream for ActiveClientStream { } // 3. Read atomic mode flag (set by grace task or stop_provider_stream) - let mode = match &self.state.send_custom_stream_flag { - Some(flag) => StreamMode::from_u8(flag.load(Ordering::Acquire)), - None => StreamMode::Inner, + let Some(mode) = (match &self.state.send_custom_stream_flag { + Some(flag) => StreamMode::try_from(flag.load(Ordering::Acquire)).ok(), + None => Some(StreamMode::Inner), + }) else { + // An unknown mode can only come from a corrupted flag. Never map it + // onto a live state such as `GracePending`; end the body safely. + debug_assert!(false, "unknown stream mode flag"); + error!( + "Unknown stream mode for {}, terminating stream", + sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) + ); + self.state.terminate_quietly(); + return Poll::Ready(None); }; // Dispatch based on the current streaming phase. @@ -665,11 +750,17 @@ impl Stream for ActiveClientStream { let http_client = app_state.http_client.load(); http_client.as_ref().clone() }; + let lifecycle = self + .state + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); let future = Box::pin(async move { - match create_provider_stream( + match open_provider_stream_with_lifecycle( &app_state.provider_stream_ctx(), &client, context.provider_stream_factory_options, + lifecycle, ) .await { @@ -727,6 +818,9 @@ impl Stream for ActiveClientStream { match self.state.inner.as_mut().map(|inner| Pin::new(inner).poll_next(cx)) { Some(Poll::Ready(Some(Ok(bytes)))) => { + if !bytes.is_empty() { + self.state.confirm_lease_once(); + } self.state.mark_direct_body_progress(); return Poll::Ready(Some(Ok(bytes))); } @@ -771,6 +865,17 @@ impl Stream for ActiveClientStream { } } + // Quiet termination: the reentry guard declined a retry of a recently + // evicted playback. End the body without a user-visible error video. + StreamMode::ReentrySuppressed => { + info!( + "Suppressing reentry retry for {}, terminating stream", + sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) + ); + self.state.terminate_quietly(); + return Poll::Ready(None); + } + // Custom video modes: serve the appropriate buffer video_mode => { if self.state.custom_video_timeout_mode != Some(video_mode) { @@ -830,8 +935,29 @@ impl Drop for ActiveClientStream { fn drop(&mut self) { self.state.release_stream_and_provider_handle_once(); } } +/// Terminal failure while admitting a playback request. Carried to the response layer so a +/// rejected request surfaces as a non-success HTTP response instead of an empty provider body. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StreamAdmissionError { + CleanupReceiverClosed, + CleanupAdmissionTimeout, + RegistrationRejected, +} + +impl From for StreamAdmissionError { + fn from(reason: ConnectionRejectionReason) -> Self { + match reason { + ConnectionRejectionReason::CleanupReceiverClosed => Self::CleanupReceiverClosed, + ConnectionRejectionReason::CleanupAdmissionTimeout => Self::CleanupAdmissionTimeout, + ConnectionRejectionReason::RegistrationFailed => Self::RegistrationRejected, + } + } +} + #[allow(clippy::too_many_lines)] -pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParams<'_>) -> BoxedProviderStream { +pub(crate) async fn create_active_client_stream( + request: ActiveClientStreamParams<'_>, +) -> Result { let ActiveClientStreamParams { mut stream_details, app_state, @@ -862,26 +988,71 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam } else { DirectBodyIdleTimeout::disabled() }; - let registered_stream = app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid, - username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint, - provider: provider_name, - stream_channel: &stream_channel, - user_agent, - session_token, - }) - .await; - let stream_uid = registered_stream.as_ref().map(|stream| stream.uid); + // A shared response already owns a guaranteed ReleaseSharedSubscriber permit + // before this registration starts. That terminal event also removes this exact user request, so reserving + // another permit from the same bounded queue would create a circular admission + // dependency under saturation. + let shared_cleanup_capability = stream_details.shared_subscriber_id.take(); + let shared_subscriber_id = + shared_cleanup_capability.as_ref().map(tuliprox_session::SharedCleanupCapability::subscriber_id); + let request_uid = shared_subscriber_id.map_or_else( + || app_state.connection_manager.next_stream_uid(), + tuliprox_core::model::SharedSubscriberId::stream_uid, + ); + let provider_request_id = stream_details + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.playback_request_id); + let connection = crate::api::model::ConnectionParams { + meter_uid, + username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint, + provider: provider_name, + stream_channel: &stream_channel, + user_agent, + session_token, + }; + let mut registered_request = if let Some(capability) = shared_cleanup_capability.filter(|_| stream_channel.shared) { + app_state + .connection_manager + .update_connection_with_uid_using_shared_cleanup( + connection, + tuliprox_session::ConnectionHistoryMode::EmitConnect, + capability, + provider_request_id, + ) + .await + } else { + app_state + .connection_manager + .update_connection_with_uid( + connection, + tuliprox_session::ConnectionHistoryMode::EmitConnect, + request_uid, + provider_request_id, + ) + .await + }; + let request_uid = registered_request.request_uid; + let display_stream_uid = registered_request.display_uid(); + let stream_uid = Some(request_uid); + if registered_request.display_stream.is_none() { + // Admission was rejected (closed cleanup receiver, bounded timeout, or failed + // registration). No provider bytes may be delivered; the acquired provider + // handle and any open provider stream are released when `request` (and its + // `stream_details`) is dropped on return. + let reason = registered_request.rejection_reason().unwrap_or(ConnectionRejectionReason::RegistrationFailed); + error!("Stream admission rejected for user {username}: {reason:?}; dropping provider handle"); + return Err(reason.into()); + } if let Some((_, _, _m_, Some(cvt))) = stream_details.stream_info.as_ref() { - app_state.connection_manager.update_stream_detail(&fingerprint.addr, *cvt).await; + app_state.connection_manager.update_stream_detail_by_uid(request_uid, *cvt).await; } let meter = if meter_stream && meter_uid != 0 { @@ -901,7 +1072,9 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam } else { stream }; - return wrap_timed_client_stream_if_needed(app_state, stream, fingerprint.addr, VirtualId::new(virtual_id)); + // The shared source owns the claim from here; disarm the registration rollback. + registered_request.commit(); + return Ok(stream); } } @@ -928,16 +1101,29 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam let waker = Arc::new(AtomicWaker::new()); let owned_session_token: Option = session_token.map(str::to_string); + // A provider-backed playback confirms its lease owner once media flows. Local + // media and streams without a provider handle carry no lease to confirm. + let lease_owner: Option> = session_token + .filter(|_| stream_details.provider_handle.is_some() || stream_details.has_deferred_provider_open()) + .map(Arc::::from); let owned_grace_ctx = stream_details.grace_resolution_context.clone(); - let mut provider_handle_preserved = stream_details.provider_handle.clone(); - // Compute deferred provider open before moving stream_details into the grace task. + let lease_request_id = stream_details + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.playback_request_id); + // Compute deferred-open metadata before taking the handle: the grace snapshot only + // needs the provider identity, while the body owns the actual release. + let has_deferred_open = stream_details.has_deferred_provider_open(); let deferred_provider_open = create_deferred_provider_open_future(app_state, &stream_details, fingerprint, &stream_channel, req_headers); let timed_stream_context = deferred_provider_open .as_ref() .and_then(|_| create_timed_stream_context(app_state, VirtualId::new(virtual_id))); + // The body owns the provider slot from here on; the managed owner releases it + // synchronously on drop, never through a lossy cleanup message. + let mut provider_handle_preserved = stream_details.provider_handle.take(); let stream_taken = stream_details.stream.take(); - let has_deferred_open = stream_details.has_deferred_provider_open(); let grace_waker = if grant_user_grace_period || stream_details.provider_grace_active { Some(Arc::clone(&waker)) } else { None }; let (grace_stop_flag, grace_task_handle) = stream_grace_period(GracePeriodParams { @@ -957,6 +1143,7 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam grace_resolution_context: owned_grace_ctx, grace_kind: Some(connection_kind), socket_bound, + shared_subscriber_id, }); let cfg = &app_state.app_config; @@ -982,8 +1169,7 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam let stream: Option = match stream_taken { None => { if !has_deferred_open { - let provider_handle = provider_handle_preserved.take(); - app_state.connection_manager.release_provider_handle(provider_handle).await; + drop(provider_handle_preserved.take()); } None } @@ -993,12 +1179,19 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam } else { stream }; - Some(wrap_timed_client_stream_if_needed(app_state, stream, fingerprint.addr, VirtualId::new(virtual_id))) + Some(wrap_timed_client_stream_if_needed( + app_state, + stream, + fingerprint.addr, + VirtualId::new(virtual_id), + display_stream_uid, + )) } }; let preempt_cancelled = provider_handle_preserved .as_ref() + .and_then(|managed| managed.handle()) .and_then(|h| h.cancel_token.as_ref()) .map(|token| Box::pin(token.clone().cancelled_owned())); @@ -1036,9 +1229,13 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam provider_error_class: None, provider_http_status: None, provider_reconnect_count: AtomicU8::new(0), + lease_owner, + lease_confirmed: false, + lease_request_id, + request_cleanup: registered_request.into_body_cleanup(), }; - ActiveClientStream { state }.boxed() + Ok(ActiveClientStream { state }.boxed()) } fn resolve_grace_period_provisioning( @@ -1077,6 +1274,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op grace_resolution_context, grace_kind, socket_bound, + shared_subscriber_id, .. } = request; let grace_period = stream_details.grace_period; @@ -1142,7 +1340,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op None => true, }; let provider_ok = match &provider_grace_check { - Some(provider_name) => !provider_manager.is_over_limit(provider_name).await, + Some(provider_name) => !provider_manager.is_over_limit(provider_name), None => true, }; if user_ok && provider_ok { @@ -1192,40 +1390,55 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op grace_kind, ) .await; - match remaining_result.admission.permission { + match remaining_result.admission.permission() { shared::model::UserConnectionPermission::Allowed | shared::model::UserConnectionPermission::GracePeriod => { // Remaining strategy succeeded β€” proceed to Inner. - stream_strategy_flag_copy.store(StreamMode::Inner as u8, Ordering::Release); + store_stream_mode(&stream_strategy_flag_copy, StreamMode::Inner); // updated stays false } shared::model::UserConnectionPermission::Exhausted => { - // Remaining strategies exhausted β€” final UserExhausted. - stream_strategy_flag_copy.store(StreamMode::UserExhausted as u8, Ordering::Release); - connection_manager - .update_stream_detail( - &fingerprint.addr, - CustomVideoStreamType::UserConnectionsExhausted, - ) - .await; - connection_manager - .shared_stream_manager - .release_connection(&fingerprint.addr, true) - .await; - info!("User connections exhausted for active clients: {username}"); + let suppressed = remaining_result.admission.is_reentry_suppressed(); + if suppressed { + // Every remaining eviction candidate is reentry-protected, + // so this is a suppressed retry of a recently evicted + // playback. End the body quietly: a visible + // connections-exhausted video would be misleading because + // the request was declined by the reentry guard, not by a + // real connection limit. + store_stream_mode(&stream_strategy_flag_copy, StreamMode::ReentrySuppressed); + info!( + "Suppressing reentry retry for recently evicted playback of user {username}" + ); + } else { + // Remaining strategies exhausted β€” final UserExhausted. + store_stream_mode(&stream_strategy_flag_copy, StreamMode::UserExhausted); + connection_manager + .update_stream_detail( + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) + .await; + info!("User connections exhausted for active clients: {username}"); + } + if let Some(id) = shared_subscriber_id { + connection_manager.shared_stream_manager.release_subscriber(id).await; + } updated = true; } } } else { // No grace context β€” immediate UserExhausted. - stream_strategy_flag_copy.store(StreamMode::UserExhausted as u8, Ordering::Release); + store_stream_mode(&stream_strategy_flag_copy, StreamMode::UserExhausted); connection_manager .update_stream_detail( &fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted, ) .await; - connection_manager.shared_stream_manager.release_connection(&fingerprint.addr, true).await; + if let Some(id) = shared_subscriber_id { + connection_manager.shared_stream_manager.release_subscriber(id).await; + } info!("User connections exhausted for active clients: {username}"); updated = true; } @@ -1234,9 +1447,9 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op if !updated { if let Some(provider_name) = provider_grace_check { - if provider_manager.is_over_limit(&provider_name).await { + if provider_manager.is_over_limit(&provider_name) { if let Some(provisioning_info) = provisioning_info { - stream_strategy_flag_copy.store(StreamMode::Provisioning as u8, Ordering::Release); + store_stream_mode(&stream_strategy_flag_copy, StreamMode::Provisioning); connection_manager .update_stream_detail(&fingerprint.addr, CustomVideoStreamType::Provisioning) .await; @@ -1261,7 +1474,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op } }); } else { - stream_strategy_flag_copy.store(StreamMode::ProviderExhausted as u8, Ordering::Release); + store_stream_mode(&stream_strategy_flag_copy, StreamMode::ProviderExhausted); connection_manager .update_stream_detail( &fingerprint.addr, @@ -1269,10 +1482,9 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op ) .await; // Release the shared stream subscription to stop the subscriber loop - connection_manager - .shared_stream_manager - .release_connection(&fingerprint.addr, true) - .await; + if let Some(id) = shared_subscriber_id { + connection_manager.shared_stream_manager.release_subscriber(id).await; + } info!("Provider connections exhausted for active clients: {provider_name}"); } updated = true; @@ -1281,7 +1493,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op } if !updated { - stream_strategy_flag_copy.store(StreamMode::Inner as u8, Ordering::Release); + store_stream_mode(&stream_strategy_flag_copy, StreamMode::Inner); } // Resolve session lifecycle transitions. @@ -1465,7 +1677,7 @@ mod tests { let event_manager = Arc::new(EventManager::new()); let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); - provider_manager.set_shared_stream_manager(Arc::clone(&shared_manager)); + provider_manager.set_shared_stream_manager(&shared_manager); let geo_ip = Arc::new(ArcSwapOption::::default()); let config = app_cfg.config.load(); @@ -1479,7 +1691,7 @@ mod tests { let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let config = app_cfg.config.load(); @@ -1558,7 +1770,7 @@ mod tests { let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let geoip = Arc::new(ArcSwapOption::::default()); let config_loaded = app_cfg.config.load(); @@ -1730,7 +1942,12 @@ mod tests { if provider_handle.is_some() { stream_details.provider_name = Some("provider_1".intern()); } - stream_details.provider_handle = provider_handle; + stream_details.provider_handle = provider_handle.map(|handle| { + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + handle, + ) + }); let virtual_id = stream_channel.virtual_id; let stream = create_active_client_stream(ActiveClientStreamParams { @@ -1747,7 +1964,8 @@ mod tests { meter_uid: 0, meter_stream: false, }) - .await; + .await + .expect("direct test stream admission should succeed"); let uid = app_state .active_users .active_streams() @@ -1772,7 +1990,6 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("direct test stream should acquire the provider slot") } @@ -1791,7 +2008,7 @@ mod tests { TestLifecycleSnapshot { active_counts, stream_uids, - provider_connections: app_state.active_provider.get_provider_connections_count().await, + provider_connections: app_state.active_provider.get_provider_connections_count(), } } @@ -1835,9 +2052,10 @@ mod tests { fn create_deferred_provider_grace_details( provider_name: &Arc, - provider_handle: crate::api::model::ProviderHandle, + provider_handle: tuliprox_session::ManagedProviderHandle, ) -> StreamDetails { StreamDetails { + shared_subscriber_id: None, stream: None, stream_info: None, provider_name: Some(Arc::clone(provider_name)), @@ -1852,6 +2070,7 @@ mod tests { provider_handle: Some(provider_handle), content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, } } @@ -1870,10 +2089,22 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("deferred client should receive provider grace allocation"); - let stream_details = create_deferred_provider_grace_details(provider_name, deferred_handle); - let deferred_provider_handle = stream_details.provider_handle.clone(); + let mut stream_details = create_deferred_provider_grace_details( + provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + deferred_handle, + ), + ); + // The grace-resolution task only reads Copy fields of `stream_details` and never + // takes ownership of the handle, so disarm it back into the raw handle the caller + // releases explicitly after the grace task completes. + let deferred_provider_handle = stream_details + .provider_handle + .take() + .and_then(|mut managed| managed.disarm()) + .expect("deferred provider handle must be retained during grace"); let test_user = create_test_user("grace-user"); let test_fingerprint = create_test_fingerprint(deferred_addr); let pending_provider_version = if let Some(token) = session_token { @@ -1898,11 +2129,12 @@ mod tests { grace_resolution_context: None, grace_kind: None, socket_bound: false, + shared_subscriber_id: None, }); ( flag.expect("provider grace should install a mode flag"), task.expect("provider grace should spawn a grace-resolution task"), - deferred_provider_handle.expect("deferred provider handle must be retained during grace"), + deferred_provider_handle, ) } @@ -1944,6 +2176,10 @@ mod tests { provider_error_class: None, provider_http_status: None, provider_reconnect_count: AtomicU8::new(0), + lease_owner: None, + lease_confirmed: false, + lease_request_id: None, + request_cleanup: None, }; let stream = ActiveClientStream { state }; @@ -1955,26 +2191,54 @@ mod tests { #[test] fn test_custom_video_type_mapping_for_grace_modes() { - assert!(matches!( + assert_eq!( ActiveClientStreamState::custom_video_type_for_mode(StreamMode::UserExhausted), - CustomVideoStreamType::UserConnectionsExhausted - )); - assert!(matches!( + Some(CustomVideoStreamType::UserConnectionsExhausted) + ); + assert_eq!( ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ProviderExhausted), - CustomVideoStreamType::ProviderConnectionsExhausted - )); - assert!(matches!( + Some(CustomVideoStreamType::ProviderConnectionsExhausted) + ); + assert_eq!( ActiveClientStreamState::custom_video_type_for_mode(StreamMode::Provisioning), - CustomVideoStreamType::Provisioning - )); - assert!(matches!( + Some(CustomVideoStreamType::Provisioning) + ); + assert_eq!( ActiveClientStreamState::custom_video_type_for_mode(StreamMode::LowPriorityPreempted), - CustomVideoStreamType::LowPriorityPreempted - )); - assert!(matches!( + Some(CustomVideoStreamType::LowPriorityPreempted) + ); + assert_eq!( ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ChannelUnavailable), - CustomVideoStreamType::ChannelUnavailable - )); + Some(CustomVideoStreamType::ChannelUnavailable) + ); + // A suppressed reentry must never map to a user-visible error clip. + assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ReentrySuppressed), None); + assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::Inner), None); + assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::GracePending), None); + } + + #[test] + fn test_stream_mode_byte_values_match_the_implicit_discriminants() { + // Discriminants are implicit (declaration order) except the 255 sentinel, so + // this pins the wire mapping against accidental reordering. + assert_eq!(StreamMode::Inner as u8, 0); + assert_eq!(StreamMode::UserExhausted as u8, 1); + assert_eq!(StreamMode::ProviderExhausted as u8, 2); + assert_eq!(StreamMode::ChannelUnavailable as u8, 3); + assert_eq!(StreamMode::Provisioning as u8, 4); + assert_eq!(StreamMode::LowPriorityPreempted as u8, 5); + assert_eq!(StreamMode::ReentrySuppressed as u8, 6); + assert_eq!(StreamMode::GracePending as u8, 255); + } + + #[test] + fn test_stream_mode_try_from_rejects_unknown_values() { + assert_eq!(StreamMode::try_from(0), Ok(StreamMode::Inner)); + assert_eq!(StreamMode::try_from(6), Ok(StreamMode::ReentrySuppressed)); + assert_eq!(StreamMode::try_from(255), Ok(StreamMode::GracePending)); + // Unknown values must not be silently mapped onto GracePending. + assert_eq!(StreamMode::try_from(7), Err(7)); + assert_eq!(StreamMode::try_from(200), Err(200)); } #[test] @@ -2039,18 +2303,17 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("holder should consume the provider's live capacity"); let (flag, grace_task, deferred_handle) = start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, None).await; assert_eq!( - StreamMode::from_u8(flag.load(Ordering::Acquire)), + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), StreamMode::GracePending, "provider grace resolution must begin in GracePending while provider capacity is exhausted" ); - app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; + app_state.connection_manager.release_provider_handle(Some(holder_handle)); let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; assert!( @@ -2058,12 +2321,12 @@ mod tests { "provider grace resolution stayed pending after capacity_notify should have fired" ); assert_eq!( - StreamMode::from_u8(flag.load(Ordering::Acquire)), + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), StreamMode::Inner, "capacity-notify should resolve provider grace from GracePending to Inner before the deadline" ); - app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); } #[tokio::test(start_paused = true)] @@ -2107,13 +2370,12 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("holder should consume the provider's live capacity"); let (_flag, grace_task, deferred_handle) = start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace")) .await; - app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; + app_state.connection_manager.release_provider_handle(Some(holder_handle)); let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; assert!(join_result.is_ok(), "grace task should finish after capacity notify"); @@ -2128,7 +2390,7 @@ mod tests { ); assert_eq!(session.permission, UserConnectionPermission::Allowed); - app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); } #[tokio::test(start_paused = true)] @@ -2172,7 +2434,6 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("holder should consume the provider's live capacity"); let (_flag, grace_task, deferred_handle) = start_deferred_provider_grace_resolution( &app_state, @@ -2193,7 +2454,7 @@ mod tests { .await .expect("replacement pending version should be created"); - app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; + app_state.connection_manager.release_provider_handle(Some(holder_handle)); let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; assert!(join_result.is_ok(), "grace task should finish after capacity notify"); @@ -2209,7 +2470,7 @@ mod tests { assert!(pending.wake_source.is_none()); assert_eq!(session.permission, UserConnectionPermission::GracePeriod); - app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); } #[tokio::test] @@ -2226,9 +2487,14 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("deferred provider allocation"); - let stream_details = create_deferred_provider_grace_details(&provider_name, provider_handle.clone()); + let stream_details = create_deferred_provider_grace_details( + &provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + provider_handle.clone(), + ), + ); assert_eq!(stream_details.content_representation, ProviderContentRepresentationMode::PreserveOrigin); let fingerprint = create_test_fingerprint(addr); let mut stream_channel = create_test_stream_channel(1, "http://provider-1.example/live/1"); @@ -2252,7 +2518,7 @@ mod tests { ); assert!(!context.provider_stream_factory_options.response_head_is_available()); drop(context); - app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + app_state.connection_manager.release_provider_handle(Some(provider_handle)); } #[tokio::test(start_paused = true)] @@ -2272,7 +2538,6 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("holder should consume the provider's live capacity"); let deferred_handle = app_state .active_provider @@ -2283,9 +2548,14 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("deferred client should receive provider grace allocation"); - let stream_details = create_deferred_provider_grace_details(&provider_name, deferred_handle.clone()); + let stream_details = create_deferred_provider_grace_details( + &provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + deferred_handle.clone(), + ), + ); let test_user = create_test_user("grace-user"); let test_fingerprint = create_test_fingerprint(deferred_addr); let stream = create_active_client_stream(ActiveClientStreamParams { @@ -2302,7 +2572,8 @@ mod tests { meter_uid: 0, meter_stream: false, }) - .await; + .await + .expect("deferred test stream admission should succeed"); pin_mut!(stream); assert!( @@ -2310,24 +2581,21 @@ mod tests { "deferred active-client-stream should park in GracePending while waiting for provider grace resolution" ); - let third_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &third_addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .await; + let third_handle = app_state.active_provider.acquire_exact_connection_with_grace( + &provider_name, + &third_addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ); assert!( third_handle.is_none(), "deferred active-client-stream should retain the deferred provider grace reservation while GracePending" ); - app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; - app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); } #[tokio::test(start_paused = true)] @@ -2347,7 +2615,6 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("holder should consume the provider's live capacity"); let deferred_handle = app_state .active_provider @@ -2358,9 +2625,14 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("deferred shared client should receive provider grace allocation"); - let stream_details = create_deferred_provider_grace_details(&provider_name, deferred_handle.clone()); + let stream_details = create_deferred_provider_grace_details( + &provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + deferred_handle.clone(), + ), + ); let test_user = create_test_user("grace-user"); let test_fingerprint = create_test_fingerprint(deferred_addr); let stream = create_active_client_stream(ActiveClientStreamParams { @@ -2377,7 +2649,8 @@ mod tests { meter_uid: 0, meter_stream: false, }) - .await; + .await + .expect("shared deferred test stream admission should succeed"); pin_mut!(stream); assert!( @@ -2385,24 +2658,21 @@ mod tests { "shared deferred active-client-stream should stay pending instead of returning an empty stream" ); - let third_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &third_addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .await; + let third_handle = app_state.active_provider.acquire_exact_connection_with_grace( + &provider_name, + &third_addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ); assert!( third_handle.is_none(), "shared deferred active-client-stream should retain the deferred provider grace reservation while pending" ); - app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; - app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); } #[tokio::test(start_paused = true)] @@ -2450,6 +2720,10 @@ mod tests { provider_error_class: None, provider_http_status: None, provider_reconnect_count: AtomicU8::new(0), + lease_owner: None, + lease_confirmed: false, + lease_request_id: None, + request_cleanup: None, }; let stream = ActiveClientStream { state }; pin_mut!(stream); @@ -2513,7 +2787,6 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await .expect("holder should consume the provider's live capacity"); let (flag, grace_task, deferred_handle) = start_deferred_provider_grace_resolution( &app_state, @@ -2524,7 +2797,7 @@ mod tests { .await; assert_eq!( - StreamMode::from_u8(flag.load(Ordering::Acquire)), + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), StreamMode::GracePending, "provider grace resolution must begin in GracePending while provider capacity is exhausted" ); @@ -2537,7 +2810,7 @@ mod tests { "grace-resolution task should complete once the deadline expires without capacity becoming available" ); assert_eq!( - StreamMode::from_u8(flag.load(Ordering::Acquire)), + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), StreamMode::ProviderExhausted, "provider grace resolution should transition from GracePending to ProviderExhausted when the deadline expires" ); @@ -2553,8 +2826,8 @@ mod tests { ); assert_eq!(session.permission, UserConnectionPermission::Exhausted); - app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; - app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); } /// Regression test: verifies that when user-grace fails and remaining strategies are @@ -2583,6 +2856,8 @@ mod tests { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), }); @@ -2749,6 +3024,7 @@ mod tests { .expect("pending version must be created for tok-second"); let stream_details = StreamDetails { + shared_subscriber_id: None, stream: None, stream_info: None, provider_name: Some(provider_name), @@ -2763,6 +3039,7 @@ mod tests { provider_handle: None, content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: Some(grace_context.clone()), + custom_reason: None, }; let (flag, grace_task) = stream_grace_period(GracePeriodParams { @@ -2782,11 +3059,12 @@ mod tests { grace_resolution_context: Some(grace_context), grace_kind: Some(crate::api::model::ConnectionKind::Soft), socket_bound: false, + shared_subscriber_id: None, }); // Grace should be pending initially. assert_eq!( - StreamMode::from_u8(flag.as_ref().unwrap().load(Ordering::Acquire)), + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), StreamMode::GracePending, "user grace should start in GracePending" ); @@ -2798,7 +3076,7 @@ mod tests { // Remaining strategies are exhausted (only GraceHoldStream was configured, no eviction). // The session should expire with UserExhausted. assert_eq!( - StreamMode::from_u8(flag.as_ref().unwrap().load(Ordering::Acquire)), + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), StreamMode::UserExhausted, "exhausted remaining strategies should result in UserExhausted" ); @@ -2834,6 +3112,233 @@ mod tests { ); } + #[tokio::test(start_paused = true)] + #[allow(clippy::too_many_lines)] + async fn test_user_grace_failure_reentry_suppression_terminates_quietly() { + // GraceHoldStream admits the request; the remaining EvictUserOldest candidate + // is reentry-protected, so the grace task must resolve to ReentrySuppressed + // instead of painting the user-visible connections-exhausted video. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 100, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), + }); + + let provider_name = "provider_1".intern(); + let first_addr: std::net::SocketAddr = "127.0.0.1:55211".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: std::net::SocketAddr = "127.0.0.1:55212".parse().unwrap_or_else(|_| unreachable!()); + let preload_addr: std::net::SocketAddr = "127.0.0.1:55213".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let second_fingerprint = create_test_fingerprint(second_addr); + let preload_fingerprint = create_test_fingerprint(preload_addr); + + let mut user = create_test_user("grace-reentry-user"); + user.max_connections = 1; + user.soft_connections = 0; + + // Two counted Normal streams make the user over limit when grace expires. + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-first", + virtual_id: 1, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/1.ts", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "grace-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &first_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(1, "http://provider-1.example/live/1.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-first"), + }) + .await; + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preload", + virtual_id: 3, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/3.ts", + addr: &preload_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 3, + username: "grace-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &preload_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(3, "http://provider-1.example/live/3.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-preload"), + }) + .await; + + // Grace session on a distinct channel; its socket-bound stream row is what + // `mark_recent_eviction_guard_for_addr` keys the reentry guard on. + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-second", + virtual_id: 2, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/2.ts", + addr: &second_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "grace-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &second_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(2, "http://provider-1.example/live/2.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-second"), + }) + .await; + + // Protect the oldest candidate (channel 1) against a retry of this channel 2. + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&second_addr, first_addr, Duration::from_secs(3)) + .await; + + let _pending_version = app_state + .active_users + .mark_pending_provider( + "grace-reentry-user", + "tok-second", + crate::api::model::PendingProviderReason::GraceHold, + 9_999, + ) + .await + .expect("pending version must be created for tok-second"); + + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies: [AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest].into(), + kind: Some(crate::api::model::ConnectionKind::Soft), + }; + let pending_version = app_state + .active_users + .pending_provider_version("grace-reentry-user", "tok-second") + .await + .expect("pending version must be created for tok-second"); + + let stream_details = StreamDetails { + shared_subscriber_id: None, + stream: None, + stream_info: None, + provider_name: Some(provider_name), + request_url: Some("http://provider-1.example/live/2.ts".intern()), + session_headers: None, + provider_session_headers: HashMap::new(), + user_agent_stream_index: None, + grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, + provider_grace_active: false, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: Some(grace_context.clone()), + custom_reason: None, + }; + + let (flag, grace_task) = stream_grace_period(GracePeriodParams { + app_state: Arc::clone(&app_state), + stream_details, + user_grace_period: true, + user: user.clone(), + fingerprint: second_fingerprint.clone(), + virtual_id: VirtualId::new(2), + session_token: Some("tok-second".to_string()), + provisioning_info: None, + waker: None, + hold_stream: true, + capacity_notify: app_state.connection_manager.capacity_notified(), + pending_provider_version: Some(pending_version), + grace_active_version: None, + grace_resolution_context: Some(grace_context), + grace_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: true, + shared_subscriber_id: None, + }); + + assert_eq!( + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), + StreamMode::GracePending, + "user grace should start in GracePending" + ); + + tokio::time::advance(Duration::from_millis(101)).await; + let _ = grace_task.expect("grace task should be spawned").await; + + assert_eq!( + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), + StreamMode::ReentrySuppressed, + "reentry-protected remaining candidate must resolve to quiet termination" + ); + + let session = app_state + .active_users + .get_and_update_user_session("grace-reentry-user", "tok-second") + .await + .expect("session must exist after grace failure"); + assert!( + matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::Expired), + "session lifecycle should be Expired after reentry-suppressed grace failure" + ); + } + #[tokio::test(start_paused = true)] async fn test_active_client_stream_immediate_provider_stream_emits_meter_batches() { let app_state = create_test_app_state(); @@ -2862,7 +3367,8 @@ mod tests { meter_uid: 55, meter_stream: true, }) - .await; + .await + .expect("metered test stream admission should succeed"); pin_mut!(stream); let first_chunk = stream.next().await; diff --git a/backend/app/src/api/model/streams/mod.rs b/backend/app/src/api/model/streams/mod.rs index e7d52e732..500429096 100644 --- a/backend/app/src/api/model/streams/mod.rs +++ b/backend/app/src/api/model/streams/mod.rs @@ -45,4 +45,5 @@ pub use crate::utils::network::request::STREAM_IDLE_TIMEOUT; pub(crate) use shared::model::CustomVideoStreamType; // Shared-stream fan-out and its buffer moved to `tuliprox-session`; // re-exported so api call sites keep their names, module paths included. +#[cfg(test)] pub(crate) use tuliprox_session::streams::buffered_stream; diff --git a/backend/app/src/api/model/streams/provider_stream.rs b/backend/app/src/api/model/streams/provider_stream.rs index 0beed4d70..a5be0eab8 100644 --- a/backend/app/src/api/model/streams/provider_stream.rs +++ b/backend/app/src/api/model/streams/provider_stream.rs @@ -1,7 +1,7 @@ use crate::{ api::model::{ stream::{BoxedProviderStream, ProviderStreamResponse}, - CleanupEvent, CustomVideoStream, ProvisioningStream, ThrottledStream, TimedClientStream, TransportStreamBuffer, + CustomVideoStream, ProvisioningStream, ThrottledStream, TimedClientStream, TransportStreamBuffer, }, model::AppConfig, }; @@ -202,7 +202,7 @@ pub fn create_panel_api_provisioning_stream_with_stop( pub fn create_custom_video_stream_response( ctx: &ProviderStreamCtx, - addr: &SocketAddr, + _addr: &SocketAddr, video_response: CustomVideoStreamType, ) -> impl axum::response::IntoResponse + Send { let config = &ctx.app_config; @@ -217,10 +217,6 @@ pub fn create_custom_video_stream_response( CustomVideoStreamType::Provisioning => create_panel_api_provisioning_stream(config, &[]), CustomVideoStreamType::HlsSessionOrLeaseExpired => create_hls_session_or_lease_expired_stream(config, &[]), } { - ctx.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProviderConnection { - addr: *addr, - video_type: video_response, - }); let mut builder = axum::response::Response::builder().status(status_code); for (key, value) in headers { builder = builder.header(key, value); @@ -234,7 +230,18 @@ pub fn create_custom_video_stream_response( // configured `custom_stream_response_error_status` (default 502) instead of a // hard-coded 403, so a reverse proxy with `proxy_intercept_errors on;` can sever // the socket. - get_custom_stream_response_error_status(config).into_response() + let status = get_custom_stream_response_error_status(config); + let mut builder = axum::response::Response::builder().status(status); + if matches!( + video_response, + CustomVideoStreamType::UserConnectionsExhausted + | CustomVideoStreamType::ProviderConnectionsExhausted + | CustomVideoStreamType::LowPriorityPreempted + | CustomVideoStreamType::UserAccountExpired + ) { + builder = builder.header("x-tuliprox-rejection", "admission_rejected"); + } + builder.body(axum::body::Body::empty()).unwrap_or_else(|_| status.into_response()) } pub fn get_header_filter_for_item_type(item_type: PlaylistItemType) -> HeaderFilter { match item_type { diff --git a/backend/app/src/api/model/streams/provider_stream_factory.rs b/backend/app/src/api/model/streams/provider_stream_factory.rs index 73d3acd12..e22003c89 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory.rs @@ -1,9 +1,8 @@ use crate::{ api::model::{ create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers, - streams::{buffered_stream::BufferedStream, client_stream::ClientStream}, - CustomVideoStreamType, ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError, - STREAM_IDLE_TIMEOUT, + streams::client_stream::ClientStream, CustomVideoStreamType, ProviderContentRepresentationMode, + ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT, }, iptv::stalker::client::validate_public_playable_url, model::{AppConfig, ConfigProvider, ReverseProxyDisabledHeaderConfig}, @@ -36,11 +35,11 @@ use std::{ error::Error as StdError, io, net::SocketAddr, - sync::Arc, + sync::{atomic::AtomicU8, Arc}, time::{Duration, Instant}, }; use tokio_util::{io::ReaderStream, sync::CancellationToken}; -use tuliprox_core::utils::request_headers::get_headers_from_request; +use tuliprox_core::{model::AllocationId, utils::request_headers::get_headers_from_request}; use tuliprox_hls::api::{ extract_hls_provider_session_headers, log_hls_origin_content_coding, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource, @@ -49,6 +48,8 @@ use tuliprox_session::{ response_headers::{provider_response_headers, ProviderResponseHeaderError}, stream_ctx::ProviderStreamCtx, stream_options::StreamOptions, + streams::{ProviderBodyOwner, ProviderBodyOwnerConfig}, + ActiveProviderManager, ManagedProviderHandle, }; use url::Url; @@ -103,6 +104,9 @@ pub struct ProviderStreamFactoryOptions { content_representation: ProviderContentRepresentationMode, response_head_availability: ProviderResponseHeadAvailability, hls_content_coding_object_kind: Option, + cancel_token: Option, + completion_token: Option, + close_reason: Option>, } pub(crate) struct ProviderStreamFactoryParams<'a> { @@ -217,9 +221,29 @@ impl ProviderStreamFactoryOptions { ProviderContentRepresentationMode::Identity => Some(HlsOriginContentCodingObjectKind::Other), ProviderContentRepresentationMode::PreserveOrigin => None, }, + cancel_token: None, + completion_token: None, + close_reason: None, } } + pub fn set_provider_handle_tokens( + &mut self, + cancel_token: Option, + completion_token: Option, + close_reason: Option>, + ) { + self.cancel_token = cancel_token; + self.completion_token = completion_token; + self.close_reason = close_reason; + } + + pub fn get_cancel_token(&self) -> Option { self.cancel_token.clone() } + + pub fn get_completion_token(&self) -> Option { self.completion_token.clone() } + + pub fn get_close_reason(&self) -> Option> { self.close_reason.clone() } + pub fn set_provider(&mut self, provider: Option>) { self.provider = provider; } pub fn apply_user_agent_stream_index(&mut self, stream_index: u64) { @@ -748,7 +772,12 @@ async fn prepare_provider_stream_response_with_context( let headers = provider_response_headers(response.headers(), context.representation)?; let response_info = Some((headers, response.status(), Some(response.url().clone()), None)); let stream = response.bytes_stream().map_err(|error| StreamError::reqwest(&error)).boxed(); - Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers }) + Ok(ProviderStreamFactoryResponse { + stream, + info: response_info, + provider_session_headers, + has_upstream_owner: false, + }) } ProviderContentRepresentationMode::Identity => { let origin_status = response.status(); @@ -787,7 +816,12 @@ async fn prepare_provider_stream_response_with_context( let headers = provider_response_headers(&decoded.headers, context.representation)?; let response_info = Some((headers, decoded.status, Some(decoded.final_url), None)); let stream = ReaderStream::new(decoded.body).map_err(|error| provider_decoded_body_error(&error)).boxed(); - Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers }) + Ok(ProviderStreamFactoryResponse { + stream, + info: response_info, + provider_session_headers, + has_upstream_owner: false, + }) } } } @@ -963,6 +997,7 @@ async fn provider_stream_request( } } +#[allow(clippy::too_many_lines)] async fn get_provider_stream( ctx: &ProviderStreamCtx, client: &reqwest::Client, @@ -973,8 +1008,29 @@ async fn get_provider_stream( let start = Instant::now(); let mut connect_err: u32 = 1; + let handle_cancel = stream_options.get_cancel_token(); while stream_options.should_continue() { - match provider_stream_request(ctx, client, stream_options).await { + let request_future = provider_stream_request(ctx, client, stream_options); + let request_res = if let Some(ref cancel) = handle_cancel { + tokio::select! { + biased; + () = cancel.cancelled() => { + debug!( + "Provider stream request cancelled during open: {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + return Err(ProviderStreamRequestFailure::Status { + status: StatusCode::BAD_GATEWAY, + provider_error_class: "cancelled", + serve_channel_unavailable: false, + }); + } + res = request_future => res, + } + } else { + request_future.await + }; + match request_res { Ok(Some(stream_response)) => { return Ok(Some(stream_response)); } @@ -1030,12 +1086,19 @@ async fn get_provider_stream( break; } connect_err += 1; - tokio::time::sleep(Duration::from_millis(50)).await; + if let Some(ref cancel) = handle_cancel { + tokio::select! { + biased; + () = cancel.cancelled() => break, + () = tokio::time::sleep(Duration::from_millis(50)) => {}, + } + } else { + tokio::time::sleep(Duration::from_millis(50)).await; + } debug_if_enabled!("Reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); } debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); stream_options.cancel_reconnect(); - ctx.connection_manager.release_provider_connection(&stream_options.addr).await; Err(ProviderStreamRequestFailure::Status { status: StatusCode::SERVICE_UNAVAILABLE, provider_error_class: "service_unavailable", @@ -1043,14 +1106,31 @@ async fn get_provider_stream( }) } +struct ProviderOpenGuard { + token: Option, + handed_off: bool, +} + +impl Drop for ProviderOpenGuard { + fn drop(&mut self) { + if !self.handed_off { + if let Some(token) = self.token.take() { + token.cancel(); + } + } + } +} + #[allow(clippy::too_many_lines)] pub async fn create_provider_stream( ctx: &ProviderStreamCtx, client: &reqwest::Client, stream_options: ProviderStreamFactoryOptions, ) -> Option { + let mut open_guard = ProviderOpenGuard { token: stream_options.get_completion_token(), handed_off: false }; match get_provider_stream(ctx, client, &stream_options).await { - Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers })) => { + Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers, .. })) => { + let continue_signal = stream_options.get_reconnect_flag_clone(); if let Some((_headers, _status, _response_url, Some(custom_video_type))) = &info { let reason = match custom_video_type { CustomVideoStreamType::ChannelUnavailable => Some(ConnectFailureReason::ChannelUnavailable), @@ -1063,32 +1143,71 @@ pub async fn create_provider_stream( if let Some(reason) = reason { record_provider_open_failure(ctx, &stream_options, reason, None, None); } + return Some(ProviderStreamFactoryResponse { + stream: ClientStream::new(init_stream, continue_signal, None, stream_options.get_url_as_str()) + .boxed(), + info, + provider_session_headers, + has_upstream_owner: false, + }); } - let continue_signal = stream_options.get_reconnect_flag_clone(); - let stream = init_stream.boxed(); - let stream = if should_wrap_provider_stream_in_buffer(&stream_options) { - BufferedStream::new( - stream, + let handle_cancel = stream_options.get_cancel_token(); + let completion_token = stream_options.get_completion_token(); + let close_reason = stream_options.get_close_reason(); + + let cancel_token = if let Some(hc) = handle_cancel { + let combined = CancellationToken::new(); + let c1 = combined.clone(); + let c2 = combined.clone(); + let hc_clone = hc.clone(); + let cs_clone = continue_signal.clone(); + tokio::spawn(async move { + tokio::select! { + biased; + () = hc_clone.cancelled() => { + cs_clone.cancel(); + c1.cancel(); + } + () = cs_clone.cancelled() => { + c2.cancel(); + } + } + }); + combined + } else { + continue_signal.clone() + }; + + let body_owner_config = if should_wrap_provider_stream_in_buffer(&stream_options) { + ProviderBodyOwnerConfig::for_buffered( stream_options.get_buffer_size(), stream_options.get_buffer_max_bytes(), - stream_options.get_reconnect_flag_clone(), - stream_options.get_url_as_str(), ) - .boxed() } else { - stream + ProviderBodyOwnerConfig::for_direct_body() }; + + open_guard.handed_off = true; + let stream = ProviderBodyOwner::new( + init_stream.boxed(), + body_owner_config, + cancel_token, + completion_token, + close_reason, + ) + .boxed(); + Some(ProviderStreamFactoryResponse { stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()) .boxed(), info, provider_session_headers, + has_upstream_owner: true, }) } Ok(None) => None, Err(failure) => { let status = failure.status(); - ctx.connection_manager.release_provider_connection(&stream_options.addr).await; record_provider_open_failure( ctx, &stream_options, @@ -1105,6 +1224,7 @@ pub async fn create_provider_stream( stream: boxed_provider_stream, info: response_info, provider_session_headers: HashMap::new(), + has_upstream_owner: false, }); } None @@ -1112,6 +1232,57 @@ pub async fn create_provider_stream( } } +#[derive(Clone)] +pub struct ProviderStreamOpenLifecycle { + manager: Arc, + allocation_id: AllocationId, + cancel_token: Option, + completion_token: Option, + close_reason: Option>, +} + +impl ProviderStreamOpenLifecycle { + pub fn from_managed(managed: &ManagedProviderHandle) -> Option { + let handle = managed.handle()?; + Some(Self { + manager: Arc::clone(managed.manager()), + allocation_id: handle.allocation_id, + cancel_token: handle.cancel_token.clone(), + completion_token: handle.completion_token.clone(), + close_reason: Some(Arc::clone(&handle.close_reason)), + }) + } + + pub fn mark_opening(&self) { self.manager.mark_opening(self.allocation_id); } + + pub fn register_body_owner(&self) -> bool { self.manager.register_body_owner(self.allocation_id) } +} + +pub async fn open_provider_stream_with_lifecycle( + ctx: &ProviderStreamCtx, + client: &reqwest::Client, + mut stream_options: ProviderStreamFactoryOptions, + lifecycle: Option, +) -> Option { + if let Some(ref lc) = lifecycle { + lc.mark_opening(); + stream_options.set_provider_handle_tokens( + lc.cancel_token.clone(), + lc.completion_token.clone(), + lc.close_reason.clone(), + ); + } + let response = create_provider_stream(ctx, client, stream_options).await?; + if response.has_upstream_owner { + if let Some(ref lc) = lifecycle { + if !lc.register_body_owner() { + return None; + } + } + } + Some(response) +} + #[cfg(test)] mod tests { use super::*; diff --git a/backend/app/src/api/model/streams/timed_client_stream.rs b/backend/app/src/api/model/streams/timed_client_stream.rs index d6358945e..bf2e18b76 100644 --- a/backend/app/src/api/model/streams/timed_client_stream.rs +++ b/backend/app/src/api/model/streams/timed_client_stream.rs @@ -5,11 +5,7 @@ use crate::{ }; use bytes::Bytes; use futures::Stream; -use shared::{ - defaults::default_kick_secs, - model::{DisconnectReason, VirtualId}, - utils::sanitize_sensitive_info, -}; +use shared::{defaults::default_kick_secs, model::VirtualId, utils::sanitize_sensitive_info}; use std::{future::Future, net::SocketAddr, pin::Pin, sync::Arc, task::Poll, time::Duration}; use tokio::time::{sleep_until, Instant, Sleep}; @@ -19,6 +15,7 @@ enum TimeoutAction { connection_manager: Arc, addr: SocketAddr, virtual_id: VirtualId, + stream_uid: Option, }, Stop, } @@ -41,6 +38,7 @@ impl TimedClientStream { duration: u32, addr: SocketAddr, virtual_id: VirtualId, + stream_uid: Option, ) -> Self { let deadline = Box::pin(sleep_until(Instant::now() + Duration::from_secs(u64::from(duration)))); Self { @@ -51,6 +49,7 @@ impl TimedClientStream { connection_manager: Arc::clone(connection_manager), addr, virtual_id, + stream_uid, }, } } @@ -69,20 +68,22 @@ impl Stream for TimedClientStream { // wakes this task exactly when the deadline fires β€” even if the upstream // provider is stalled and emitting no data. if self.deadline.as_mut().poll(cx).is_ready() { - if let TimeoutAction::Kick { app_config, connection_manager, addr, virtual_id } = &self.timeout_action { + if let TimeoutAction::Kick { app_config, connection_manager, addr, virtual_id, stream_uid } = + &self.timeout_action + { let kick_secs = app_config.config.load().web_ui.as_ref().map_or_else(default_kick_secs, |wc| wc.kick_secs); let connection_manager = Arc::clone(connection_manager); - let addr = *addr; + let stream_uid = *stream_uid; let virtual_id = *virtual_id; debug_if_enabled!( "TimedClient stream exceeds time limit. Closing stream with virtual_id {virtual_id} for addr: {}", sanitize_sensitive_info(&addr.to_string()) ); tokio::spawn(async move { - let _ = connection_manager - .close_connection_with_reason_and_block(&addr, virtual_id, kick_secs, DisconnectReason::Timeout) - .await; + if let Some(uid) = stream_uid { + connection_manager.block_stream_by_uid(uid, virtual_id, kick_secs).await; + } }); } return Poll::Ready(None); diff --git a/backend/app/src/api/panel_api/mod.rs b/backend/app/src/api/panel_api/mod.rs index 727ba9d59..fbc5d0fc3 100644 --- a/backend/app/src/api/panel_api/mod.rs +++ b/backend/app/src/api/panel_api/mod.rs @@ -1242,10 +1242,7 @@ pub(crate) fn can_provision_on_exhausted(app_state: &AppState, input: &ConfigInp debug_if_enabled!("panel_api config invalid: {}", sanitize_sensitive_info(&err.to_string())); return false; } - if is_alias_pool_max_reached(app_state, input) { - return false; - } - true + !is_alias_pool_max_reached(app_state, input) } pub(crate) fn find_input_by_provider_name(app_state: &AppState, provider_name: &str) -> Option> { @@ -3677,6 +3674,7 @@ pub fn create_panel_api_provisioning_stream_details( ); let (stream, stream_info) = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); return StreamDetails { + shared_subscriber_id: None, stream, stream_info, provider_name, @@ -3691,6 +3689,7 @@ pub fn create_panel_api_provisioning_stream_details( provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, }; } @@ -3706,6 +3705,7 @@ pub fn create_panel_api_provisioning_stream_details( }); StreamDetails { + shared_subscriber_id: None, stream, stream_info, provider_name, @@ -3720,6 +3720,7 @@ pub fn create_panel_api_provisioning_stream_details( provider_handle: None, content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, } } diff --git a/backend/app/src/api/serve.rs b/backend/app/src/api/serve.rs index 75b0f8e4c..fcd0d9068 100644 --- a/backend/app/src/api/serve.rs +++ b/backend/app/src/api/serve.rs @@ -283,7 +283,8 @@ mod tests { meter_uid: 0, meter_stream: false, }) - .await; + .await + .expect("test stream admission should succeed"); Response::new(Body::from_stream(stream)) } @@ -349,7 +350,7 @@ mod tests { }) .await .expect("socket disconnect should release registry state and drop the upstream"); - assert_eq!(app_state.active_provider.active_connections().await.unwrap_or_default().values().sum::(), 0); + assert_eq!(app_state.active_provider.active_connections().unwrap_or_default().values().sum::(), 0); server_cancel.cancel(); tokio::time::timeout(Duration::from_secs(1), server) diff --git a/backend/app/src/main.rs b/backend/app/src/main.rs index 19173cf3f..6a718751f 100644 --- a/backend/app/src/main.rs +++ b/backend/app/src/main.rs @@ -531,6 +531,7 @@ async fn start_in_cli_mode(cfg: Arc, targets: Arc) { } async fn start_in_server_mode(cfg: Arc, targets: Arc) { + tuliprox_core::utils::runtime_liveness::start(&tokio::runtime::Handle::current()); if let Err(err) = api::main_api::start_server(cfg, targets).await { exit!("Can't start server: {err}"); } diff --git a/backend/btree/src/v3/migration.rs b/backend/btree/src/v3/migration.rs index e8723df07..aaf5f2a5e 100644 --- a/backend/btree/src/v3/migration.rs +++ b/backend/btree/src/v3/migration.rs @@ -501,7 +501,7 @@ mod tests { let mut bytes = std::fs::read(&path)?; let serialized = crate::codec::binary_serialize(&large2)?; let mut pattern = vec![v2::COMPRESSION_FLAG_LZ4]; - pattern.extend_from_slice(&(serialized.len() as u32).to_le_bytes()); + pattern.extend_from_slice(&u32::try_from(serialized.len()).expect("test record length fits u32").to_le_bytes()); // Find the second occurrence (key 2) let mut occurrences = Vec::new(); for pos in 0..bytes.len().saturating_sub(pattern.len()) { diff --git a/backend/core/src/model/config/stream.rs b/backend/core/src/model/config/stream.rs index 2b93baf92..6a439259c 100644 --- a/backend/core/src/model/config/stream.rs +++ b/backend/core/src/model/config/stream.rs @@ -1,8 +1,10 @@ use crate::model::macros; use shared::{ + defaults::DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS, model::{AdmissionStrategy, StreamBufferConfigDto, StreamConfigDto}, utils::parse_to_kbps, }; +use std::time::Duration; use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; #[derive(Debug, Clone)] @@ -40,6 +42,8 @@ pub struct StreamConfig { pub throttle_kbps: u64, pub shared_burst_buffer_mb: u64, pub shared_subscriber_idle_timeout_secs: u64, + pub cleanup_queue_capacity: usize, + pub recent_eviction_reentry_ttl: Duration, pub admission_strategies: Option>, } @@ -59,6 +63,8 @@ impl Default for StreamConfig { throttle_kbps: 0, shared_burst_buffer_mb: 12, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: Duration::from_millis(DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS), admission_strategies: None, } } @@ -78,6 +84,8 @@ impl From<&StreamConfigDto> for StreamConfig { throttle_kbps: dto.throttle.as_ref().map_or(0u64, |throttle| parse_to_kbps(throttle).unwrap_or(0u64)), shared_burst_buffer_mb: dto.shared_burst_buffer_mb, shared_subscriber_idle_timeout_secs: dto.shared_subscriber_idle_timeout_secs, + cleanup_queue_capacity: dto.cleanup_queue_capacity, + recent_eviction_reentry_ttl: Duration::from_millis(dto.recent_eviction_reentry_ttl_ms), admission_strategies: dto.admission_strategies.clone(), } } @@ -98,6 +106,12 @@ impl From<&StreamConfig> for StreamConfigDto { throttle_kbps: instance.throttle_kbps, shared_burst_buffer_mb: instance.shared_burst_buffer_mb, shared_subscriber_idle_timeout_secs: instance.shared_subscriber_idle_timeout_secs, + cleanup_queue_capacity: instance.cleanup_queue_capacity, + recent_eviction_reentry_ttl_ms: instance + .recent_eviction_reentry_ttl + .as_millis() + .try_into() + .unwrap_or(DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS), admission_strategies: instance.admission_strategies.clone(), } } @@ -108,6 +122,18 @@ mod tests { use super::StreamConfig; use shared::model::{AdmissionStrategy, StreamConfigDto}; + #[test] + fn stream_config_defaults_reentry_ttl_to_the_shared_default() { + let config = StreamConfig::default(); + assert_eq!( + config.recent_eviction_reentry_ttl, + std::time::Duration::from_millis(shared::defaults::DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS) + ); + // The documented default is 3000 ms for backward compatibility. + assert_eq!(shared::defaults::DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS, 3000); + assert_eq!(shared::defaults::default_recent_eviction_reentry_ttl_ms(), 3000); + } + #[test] fn stream_config_preserves_missing_admission_strategies() { let domain = StreamConfig::from(&StreamConfigDto::default()); @@ -136,6 +162,8 @@ mod tests { throttle_kbps: 0, shared_burst_buffer_mb: 1, shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(2_500), admission_strategies: Some(vec![ AdmissionStrategy::EvictUserOldest, AdmissionStrategy::GraceHoldStream, @@ -144,6 +172,7 @@ mod tests { }; let dto = StreamConfigDto::from(&domain); + assert_eq!(dto.recent_eviction_reentry_ttl_ms, 2_500); assert_eq!( dto.admission_strategies, Some(vec![ diff --git a/backend/core/src/model/healthcheck.rs b/backend/core/src/model/healthcheck.rs index 46b5534c3..7009de853 100644 --- a/backend/core/src/model/healthcheck.rs +++ b/backend/core/src/model/healthcheck.rs @@ -8,4 +8,32 @@ pub struct Healthcheck { #[serde(default, skip_serializing_if = "is_blank_optional_string")] pub build_time: Option, pub server_time: String, + /// Runtime liveness as observed by the scheduler heartbeat. Absent before + /// the watchdog is installed (for example in setup mode). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime: Option, +} + +/// Liveness snapshot produced by the scheduler heartbeat. +/// +/// `status` is `alive` while the heartbeat is within its stall threshold and +/// `stalled` otherwise. This field is informational: the enclosing +/// [`Healthcheck::status`] stays `ok` for as long as the HTTP server answers, +/// so an orchestrator does not restart a process that still serves requests. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RuntimeHealth { + pub status: String, + pub heartbeat_age_ms: u64, + pub heartbeat_interval_ms: u64, + pub stall_threshold_ms: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stalled_for_ms: Option, + pub ticks: u64, + pub stall_episodes: u64, + pub max_schedule_delay_ms: u64, + pub uptime_ms: u64, + /// `true` when `TULIPROX_WATCHDOG=2` armed the watchdog to restart the + /// process on a confirmed stall. + #[serde(default)] + pub restart_on_stall: bool, } diff --git a/backend/core/src/model/mod.rs b/backend/core/src/model/mod.rs index 31f60a11b..06570ad01 100644 --- a/backend/core/src/model/mod.rs +++ b/backend/core/src/model/mod.rs @@ -17,6 +17,7 @@ pub mod auth_rejection; pub mod batch_result_collector; pub mod custom_stream_flags; pub mod fingerprint; +pub mod playback; pub mod playlist_filter; pub mod playlist_key; pub mod provider; @@ -32,8 +33,8 @@ pub mod xtream_response; pub use self::{ auth_rejection::*, batch_result_collector::*, config::*, custom_stream_flags::*, fingerprint::*, healthcheck::*, - input_source::*, mapping::*, messaging::*, notification::*, playlist_filter::*, playlist_key::*, provider::*, - proxy_redirect::*, stalker_record::*, stream_error::*, stream_history::*, target_bouquet::*, update_guard::*, - update_quality::*, update_task::*, xmltv::*, xtream::*, + input_source::*, mapping::*, messaging::*, notification::*, playback::*, playlist_filter::*, playlist_key::*, + provider::*, proxy_redirect::*, stalker_record::*, stream_error::*, stream_history::*, target_bouquet::*, + update_guard::*, update_quality::*, update_task::*, xmltv::*, xtream::*, }; pub use shared::model::xtream_const::*; diff --git a/backend/core/src/model/playback.rs b/backend/core/src/model/playback.rs new file mode 100644 index 000000000..c455cc019 --- /dev/null +++ b/backend/core/src/model/playback.rs @@ -0,0 +1,278 @@ +//! Request, provider-lease and shared-subscriber identities. + +use shared::{ + defaults::{DASH_EXT, HLS_EXT}, + model::PlaylistItemType, +}; +use std::{ + fmt, + sync::atomic::{AtomicU64, Ordering}, +}; + +static NEXT_REQUEST_ID: AtomicU64 = AtomicU64::new(1); +static NEXT_LEASE_ID: AtomicU64 = AtomicU64::new(1); + +/// Identity of one shared-stream subscriber, allocated from the user-stream UID sequence. +/// +/// A shared stream fans one provider connection out to several clients. Keying those +/// subscribers by socket address collapses two external clients that arrive through +/// one reverse-proxy socket into a single entry, so the second subscription replaces +/// and cancels the first. Subscribers therefore own a unique id and the socket stays +/// display metadata plus an explicit socket-wide close target. +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct SharedSubscriberId(u32); + +impl SharedSubscriberId { + #[inline] + pub const fn from_stream_uid(value: u32) -> Self { Self(value) } + + #[inline] + pub const fn stream_uid(self) -> u32 { self.0 } +} + +impl fmt::Display for SharedSubscriberId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "sub#{}", self.0) } +} + +/// Server-side unique number of one playback request. +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct PlaybackRequestId(u64); + +impl PlaybackRequestId { + #[inline] + pub fn next() -> Self { Self(NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed)) } + + /// Constructs a caller-supplied id for tests or diagnostics. + #[inline] + pub const fn from_raw(value: u64) -> Self { Self(value) } + + #[inline] + pub const fn as_u64(self) -> u64 { self.0 } +} + +impl fmt::Display for PlaybackRequestId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "req#{}", self.0) } +} + +/// Identifier of one logical provider capacity slot lease. +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct PlaybackLeaseId(u64); + +impl PlaybackLeaseId { + #[inline] + pub fn next() -> Self { Self(NEXT_LEASE_ID.fetch_add(1, Ordering::Relaxed)) } + + #[inline] + pub const fn from_raw(value: u64) -> Self { Self(value) } + + #[inline] + pub const fn as_u64(self) -> u64 { self.0 } +} + +impl fmt::Display for PlaybackLeaseId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "lease#{}", self.0) } +} + +/// Unambiguous identity of one provider binding incarnation. +/// +/// `lease_id` alone survives remove/recreate, while `generation` distinguishes a +/// same-account rebind that reuses the same lease. Together they let a delayed +/// detach/clear prove it still targets the exact binding it originally acquired, +/// instead of the current owner row (which may already be a successor). +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +pub struct ProviderBindingTag { + pub lease_id: PlaybackLeaseId, + pub generation: u64, +} + +impl ProviderBindingTag { + #[inline] + pub const fn new(lease_id: PlaybackLeaseId, generation: u64) -> Self { Self { lease_id, generation } } +} + +/// Media class of a playback. Part of the family key, so a live TS playback and a +/// catchup playback of the same channel never share a lease. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum PlaybackKind { + LiveTs, + LiveHls, + LiveDash, + Vod, + Series, + Catchup, + /// Background work (probe, metadata, download, recording). Never reconnect-capable. + Internal, +} + +impl PlaybackKind { + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + Self::LiveTs => "live_ts", + Self::LiveHls => "live_hls", + Self::LiveDash => "live_dash", + Self::Vod => "vod", + Self::Series => "series", + Self::Catchup => "catchup", + Self::Internal => "internal", + } + } + + /// True when an interrupted playback of this kind may keep its provider slot + /// for a short reconnect window instead of releasing it immediately. + #[inline] + pub const fn is_reconnect_capable(self) -> bool { + matches!(self, Self::LiveHls | Self::LiveDash | Self::Vod | Self::Series | Self::Catchup) + } + + /// Classifies a playlist item plus its effective playback extension. + /// + /// The extension matters: a live item served as `.m3u8` is an adaptive playback + /// whose reconnects arrive as separate requests, while the same item served as + /// `.ts` is one long response. + pub fn classify(item_type: PlaylistItemType, extension: Option<&str>) -> Self { + let is_hls = extension.is_some_and(|ext| ext.eq_ignore_ascii_case(HLS_EXT)); + let is_dash = extension.is_some_and(|ext| ext.eq_ignore_ascii_case(DASH_EXT)); + match item_type { + PlaylistItemType::Catchup => Self::Catchup, + PlaylistItemType::Video | PlaylistItemType::LocalVideo => Self::Vod, + PlaylistItemType::Series + | PlaylistItemType::SeriesInfo + | PlaylistItemType::LocalSeries + | PlaylistItemType::LocalSeriesInfo => Self::Series, + PlaylistItemType::LiveDash => Self::LiveDash, + PlaylistItemType::LiveHls => Self::LiveHls, + PlaylistItemType::Live | PlaylistItemType::LiveUnknown => { + if is_dash { + Self::LiveDash + } else if is_hls { + Self::LiveHls + } else { + Self::LiveTs + } + } + } + } +} + +impl fmt::Display for PlaybackKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } +} + +/// Why a playback request ended. Each outcome has an explicit lease policy. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PlaybackRequestOutcome { + Completed, + ClientClosed, + FailedBeforeMedia, + ProviderFailed, + Preempted, + Kicked, + TimedOut, + SessionExpired, + ServerShutdown, +} + +impl PlaybackRequestOutcome { + /// True when the lease may stay as a reconnect-capable idle lease. + /// + /// Only a clean end of a playback that already produced media keeps the slot; + /// failures and forced terminations release capacity immediately. + #[inline] + pub const fn keeps_idle_lease(self) -> bool { matches!(self, Self::Completed | Self::ClientClosed) } + + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + Self::Completed => "completed", + Self::ClientClosed => "client_closed", + Self::FailedBeforeMedia => "failed_before_media", + Self::ProviderFailed => "provider_failed", + Self::Preempted => "preempted", + Self::Kicked => "kicked", + Self::TimedOut => "timed_out", + Self::SessionExpired => "session_expired", + Self::ServerShutdown => "server_shutdown", + } + } +} + +impl fmt::Display for PlaybackRequestOutcome { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } +} + +/// Explains a provider selection or rejection decision in logs and metrics. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PlaybackSelectionReason { + NewPriorityAllocation, + SameLeaseReattach, + SharedOriginReuse, + HigherPriorityUnavailable, + ReservedCapacity, + Grace, + Preemption, + LeaseExpired, + StartupAbandoned, +} + +impl PlaybackSelectionReason { + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + Self::NewPriorityAllocation => "new-priority-allocation", + Self::SameLeaseReattach => "same-lease-reattach", + Self::SharedOriginReuse => "shared-origin-reuse", + Self::HigherPriorityUnavailable => "higher-priority-unavailable", + Self::ReservedCapacity => "reserved-capacity", + Self::Grace => "grace", + Self::Preemption => "preemption", + Self::LeaseExpired => "lease-expired", + Self::StartupAbandoned => "startup-abandoned", + } + } +} + +impl fmt::Display for PlaybackSelectionReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_and_lease_ids_are_unique() { + let first = PlaybackRequestId::next(); + let second = PlaybackRequestId::next(); + assert_ne!(first, second); + assert_ne!(PlaybackLeaseId::next(), PlaybackLeaseId::next()); + } + + #[test] + fn live_extension_selects_adaptive_kind() { + assert_eq!(PlaybackKind::classify(PlaylistItemType::Live, Some(".ts")), PlaybackKind::LiveTs); + assert_eq!(PlaybackKind::classify(PlaylistItemType::Live, Some(".m3u8")), PlaybackKind::LiveHls); + assert_eq!(PlaybackKind::classify(PlaylistItemType::LiveUnknown, Some(".mpd")), PlaybackKind::LiveDash); + assert_eq!(PlaybackKind::classify(PlaylistItemType::Catchup, None), PlaybackKind::Catchup); + assert_eq!(PlaybackKind::classify(PlaylistItemType::Video, None), PlaybackKind::Vod); + assert_eq!(PlaybackKind::classify(PlaylistItemType::Series, None), PlaybackKind::Series); + } + + #[test] + fn only_clean_outcomes_keep_idle_leases() { + assert!(PlaybackRequestOutcome::Completed.keeps_idle_lease()); + assert!(PlaybackRequestOutcome::ClientClosed.keeps_idle_lease()); + assert!(!PlaybackRequestOutcome::FailedBeforeMedia.keeps_idle_lease()); + assert!(!PlaybackRequestOutcome::Preempted.keeps_idle_lease()); + assert!(!PlaybackRequestOutcome::TimedOut.keeps_idle_lease()); + assert!(!PlaybackRequestOutcome::SessionExpired.keeps_idle_lease()); + assert!(!PlaybackRequestOutcome::ServerShutdown.keeps_idle_lease()); + } + + #[test] + fn live_ts_is_not_reconnect_capable() { + assert!(!PlaybackKind::LiveTs.is_reconnect_capable()); + assert!(PlaybackKind::LiveHls.is_reconnect_capable()); + assert!(!PlaybackKind::Internal.is_reconnect_capable()); + } +} diff --git a/backend/core/src/model/provider.rs b/backend/core/src/model/provider.rs index 9466d0358..97c7d6660 100644 --- a/backend/core/src/model/provider.rs +++ b/backend/core/src/model/provider.rs @@ -11,8 +11,14 @@ use crate::{ }; use log::debug; use shared::{model::InputType, utils::sanitize_sensitive_info, write_if_some}; -use std::{fmt, net::SocketAddr, ops::Deref, sync::Arc, time::Duration}; -use tokio::{sync::RwLock, time::Instant}; +use std::{ + fmt, + net::SocketAddr, + ops::Deref, + sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, + time::Duration, +}; +use tokio::time::Instant; use tokio_util::sync::CancellationToken; /// Address of the client holding a provider connection. @@ -73,11 +79,11 @@ impl ProviderAllocation { } } - pub async fn release(&self) { + pub fn release(&self) { match &self { ProviderAllocation::Exhausted => {} ProviderAllocation::Available(config) | ProviderAllocation::GracePeriod(config) => { - config.release().await; + config.release(); } } } @@ -88,13 +94,57 @@ impl ProviderAllocation { } } +use std::sync::atomic::{AtomicU8, Ordering}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectionLifecycle { + Opening, + Active, + Closing, + Closed, +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderCloseReason { + Unspecified = 0, + Superseded = 1, + PriorityPreempted = 2, + ClientClosed = 3, + IdleTimeout = 4, + ProviderError = 5, + Shutdown = 6, +} + +impl ProviderCloseReason { + pub fn from_u8(v: u8) -> Self { + match v { + 1 => Self::Superseded, + 2 => Self::PriorityPreempted, + 3 => Self::ClientClosed, + 4 => Self::IdleTimeout, + 5 => Self::ProviderError, + 6 => Self::Shutdown, + _ => Self::Unspecified, + } + } +} + #[derive(Debug, Clone)] pub struct ProviderHandle { + pub playback_request_id: Option, + /// Exact provider binding incarnation this handle was acquired under. A delayed + /// detach/clear must present this tag to release the slot; a bare generation or + /// owner lookup is not enough because a recreated lease can reuse generation `1`. + pub binding_tag: Option, pub client_id: ClientConnectionId, pub allocation_id: AllocationId, pub allocation: ProviderAllocation, // Token to cancel the background task (e.g. internal probe) if preempted pub cancel_token: Option, + pub completion_token: Option, + pub close_reason: Arc, + pub open_generation: u64, } impl ProviderHandle { @@ -104,7 +154,30 @@ impl ProviderHandle { allocation: ProviderAllocation, cancel_token: Option, ) -> Self { - Self { client_id, allocation_id, allocation, cancel_token } + Self { + client_id, + allocation_id, + allocation, + cancel_token, + completion_token: None, + close_reason: Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8)), + playback_request_id: None, + binding_tag: None, + open_generation: 0, + } + } + + pub fn set_close_reason(&self, reason: ProviderCloseReason) { + let _ = self.close_reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + reason as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + + pub fn get_close_reason(&self) -> ProviderCloseReason { + ProviderCloseReason::from_u8(self.close_reason.load(Ordering::Acquire)) } } @@ -185,15 +258,39 @@ impl PartialEq for ProviderConfig { macro_rules! modify_connections { ($self:ident, $guard:ident, +1) => {{ $guard.current_connections += 1; - $self.notify_connection_change($guard.current_connections); + let count = $guard.current_connections; + drop($guard); + $self.notify_connection_change(count); }}; ($self:ident, $guard:ident, -1) => {{ $guard.current_connections = $guard.current_connections.saturating_sub(1); - $self.notify_connection_change($guard.current_connections); + let count = $guard.current_connections; + drop($guard); + $self.notify_connection_change(count); }}; } impl ProviderConfig { + fn read_connection(&self) -> RwLockReadGuard<'_, ProviderConfigConnection> { + match self.connection.read() { + Ok(guard) => guard, + Err(poisoned) => { + log::warn!("Recovering poisoned provider connection read lock for {}", self.name); + poisoned.into_inner() + } + } + } + + fn write_connection(&self) -> RwLockWriteGuard<'_, ProviderConfigConnection> { + match self.connection.write() { + Ok(guard) => guard, + Err(poisoned) => { + log::warn!("Recovering poisoned provider connection write lock for {}", self.name); + poisoned.into_inner() + } + } + } + fn grace_is_active(connection: &ProviderConfigConnection, now: Instant, grace_period_timeout_secs: u64) -> bool { connection.grace_started_at.is_some_and(|started_at| { now.checked_duration_since(started_at).unwrap_or_default() <= Duration::from_secs(grace_period_timeout_secs) @@ -280,25 +377,21 @@ impl ProviderConfig { } #[inline] - pub async fn is_exhausted(&self) -> bool { + pub fn is_exhausted(&self) -> bool { let max = self.max_connections; if max == 0 { return false; } - self.connection.read().await.current_connections >= max + self.read_connection().current_connections >= max } #[inline] - pub async fn is_over_limit(&self, grace_period_timeout_secs: u64) -> bool { + pub fn is_over_limit(&self, grace_period_timeout_secs: u64) -> bool { let max = self.max_connections; if max == 0 { return false; } - let mut guard = self.connection.write().await; - if guard.current_connections < self.max_connections { - guard.grace_started_at = None; - } - + let guard = self.read_connection(); if guard.current_connections > max { if Self::grace_is_active(&guard, Instant::now(), grace_period_timeout_secs) { // Grace timeout still active, deny connection @@ -306,6 +399,13 @@ impl ProviderConfig { } return true; } + if guard.current_connections < max && guard.grace_started_at.is_some() { + drop(guard); + let mut write_guard = self.write_connection(); + if write_guard.current_connections < max { + write_guard.grace_started_at = None; + } + } false } @@ -315,12 +415,12 @@ impl ProviderConfig { // !self.is_exhausted() // } - async fn try_allocate(&self, grace: bool, grace_period_timeout_secs: u64) -> ProviderConfigAllocation { + fn try_allocate(&self, grace: bool, grace_period_timeout_secs: u64) -> ProviderConfigAllocation { if is_input_expired(self.exp_date) { return ProviderConfigAllocation::Exhausted; } - let mut guard = self.connection.write().await; + let mut guard = self.write_connection(); if self.max_connections == 0 { modify_connections!(self, guard, +1); return ProviderConfigAllocation::Available; @@ -353,7 +453,7 @@ impl ProviderConfig { // is intended to use with redirects, to cycle through provider // do not increment and connection counter! - async fn get_next(&self, grace: bool, grace_period_timeout_secs: u64) -> bool { + fn get_next(&self, grace: bool, grace_period_timeout_secs: u64) -> bool { if is_input_expired(self.exp_date) { return false; } @@ -361,7 +461,7 @@ impl ProviderConfig { if self.max_connections == 0 { return true; } - let mut guard = self.connection.write().await; + let mut guard = self.write_connection(); let connections = guard.current_connections; if connections < self.max_connections { guard.grace_started_at = None; @@ -382,8 +482,8 @@ impl ProviderConfig { true } - pub async fn release(&self) { - let mut guard = self.connection.write().await; + pub fn release(&self) { + let mut guard = self.write_connection(); // Releasing while over capacity ends the single outstanding grace grant. // Unlimited providers do not use grace tracking. if self.max_connections > 0 && guard.current_connections > self.max_connections { @@ -395,7 +495,7 @@ impl ProviderConfig { } #[inline] - pub async fn get_current_connections(&self) -> usize { self.connection.read().await.current_connections } + pub fn get_current_connections(&self) -> usize { self.read_connection().current_connections } #[inline] pub fn get_priority(&self) -> i16 { self.priority } @@ -415,16 +515,16 @@ impl ProviderConfigWrapper { pub fn config(&self) -> Arc { Arc::clone(&self.inner) } - pub async fn try_allocate(&self, grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { - match self.inner.try_allocate(grace, grace_period_timeout_secs).await { + pub fn try_allocate(&self, grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { + match self.inner.try_allocate(grace, grace_period_timeout_secs) { ProviderConfigAllocation::Available => ProviderAllocation::new_available(Arc::clone(&self.inner)), ProviderConfigAllocation::GracePeriod => ProviderAllocation::new_grace_period(Arc::clone(&self.inner)), ProviderConfigAllocation::Exhausted => ProviderAllocation::Exhausted, } } - pub async fn get_next(&self, grace: bool, grace_period_timeout_secs: u64) -> Option> { - if self.inner.get_next(grace, grace_period_timeout_secs).await { + pub fn get_next(&self, grace: bool, grace_period_timeout_secs: u64) -> Option> { + if self.inner.get_next(grace, grace_period_timeout_secs) { return Some(Arc::clone(&self.inner)); } None @@ -504,59 +604,76 @@ mod tests { assert!(provider.is_unlimited()); // Drive connection count up so `release` executes its normal path. - assert!(matches!(provider.try_allocate(false, 0).await, ProviderConfigAllocation::Available)); - assert!(matches!(provider.try_allocate(false, 0).await, ProviderConfigAllocation::Available)); - assert_eq!(provider.get_current_connections().await, 2); + assert!(matches!(provider.try_allocate(false, 0), ProviderConfigAllocation::Available)); + assert!(matches!(provider.try_allocate(false, 0), ProviderConfigAllocation::Available)); + assert_eq!(provider.get_current_connections(), 2); { - let mut guard = provider.connection.write().await; + let mut guard = provider.write_connection(); guard.grace_started_at = Some(Instant::now()); } - provider.release().await; + provider.release(); - let guard = provider.connection.read().await; + let guard = provider.read_connection(); assert_eq!(guard.current_connections, 1, "release must still decrement connection count"); assert!(guard.grace_started_at.is_some(), "release must not clear grace state for unlimited provider"); } #[tokio::test] async fn concurrent_allocations_grant_only_one_grace_connection() { - let provider = build_test_config(1); - assert!(matches!(provider.try_allocate(false, 10).await, ProviderConfigAllocation::Available)); + let provider = Arc::new(build_test_config(1)); + assert!(matches!(provider.try_allocate(false, 10), ProviderConfigAllocation::Available)); - let (first, second) = tokio::join!(provider.try_allocate(true, 10), provider.try_allocate(true, 10)); - let grace_count = [first, second] - .into_iter() + let barrier = Arc::new(std::sync::Barrier::new(3)); + let first_provider = Arc::clone(&provider); + let first_barrier = Arc::clone(&barrier); + let first = std::thread::spawn(move || { + first_barrier.wait(); + first_provider.try_allocate(true, 10) + }); + let second_provider = Arc::clone(&provider); + let second_barrier = Arc::clone(&barrier); + let second = std::thread::spawn(move || { + second_barrier.wait(); + second_provider.try_allocate(true, 10) + }); + barrier.wait(); + let allocations = [first.join().ok(), second.join().ok()]; + assert!(allocations.iter().all(Option::is_some)); + let grace_count = allocations + .iter() + .flatten() .filter(|allocation| matches!(allocation, ProviderConfigAllocation::GracePeriod)) .count(); - let exhausted_count = [first, second] - .into_iter() + let exhausted_count = allocations + .iter() + .flatten() .filter(|allocation| matches!(allocation, ProviderConfigAllocation::Exhausted)) .count(); assert_eq!(grace_count, 1); assert_eq!(exhausted_count, 1); - assert_eq!(provider.get_current_connections().await, 2); + assert_eq!(provider.get_current_connections(), 2); } #[tokio::test(start_paused = true)] async fn active_grace_at_limit_blocks_another_grant_until_timeout() { let provider = build_test_config(1); { - let mut guard = provider.connection.write().await; + let mut guard = provider.write_connection(); guard.current_connections = 1; guard.grace_started_at = Some(Instant::now()); } - assert!(matches!(provider.try_allocate(true, 10).await, ProviderConfigAllocation::Exhausted)); - assert!(!provider.get_next(true, 10).await); - assert!(provider.connection.read().await.grace_started_at.is_some()); + assert!(matches!(provider.try_allocate(true, 10), ProviderConfigAllocation::Exhausted)); + assert!(!provider.get_next(true, 10)); + assert!(provider.read_connection().grace_started_at.is_some()); tokio::time::advance(Duration::from_secs(11)).await; - assert!(provider.get_next(true, 10).await); - assert!(provider.connection.read().await.grace_started_at.is_none()); - assert!(matches!(provider.try_allocate(true, 10).await, ProviderConfigAllocation::GracePeriod)); + assert!(provider.get_next(true, 10)); + assert!(provider.read_connection().grace_started_at.is_none()); + assert!(matches!(provider.try_allocate(true, 10), ProviderConfigAllocation::GracePeriod)); } } diff --git a/backend/core/src/model/target_bouquet.rs b/backend/core/src/model/target_bouquet.rs index 5bfec31dc..70e0e5346 100644 --- a/backend/core/src/model/target_bouquet.rs +++ b/backend/core/src/model/target_bouquet.rs @@ -76,7 +76,7 @@ mod tests { #[test] fn case_sensitive_exact_matching() { let filter = TargetBouquetFilter::from_dto(whitelist(PlaylistClusterBouquetDto { - live: Some(vec!["news".to_string(), "".to_string()]), + live: Some(vec!["news".to_string(), String::new()]), vod: None, series: None, })) diff --git a/backend/core/src/utils/clock.rs b/backend/core/src/utils/clock.rs index a8dc5df58..791a4d824 100644 --- a/backend/core/src/utils/clock.rs +++ b/backend/core/src/utils/clock.rs @@ -79,7 +79,14 @@ impl ManualClock { /// Move time forward, saturating rather than wrapping. pub fn advance(&self, delta_ms: u64) { - self.0.fetch_update(Ordering::Release, Ordering::Acquire, |now| Some(now.saturating_add(delta_ms))).ok(); + let mut now = self.0.load(Ordering::Acquire); + loop { + match self.0.compare_exchange_weak(now, now.saturating_add(delta_ms), Ordering::Release, Ordering::Acquire) + { + Ok(_) => break, + Err(observed) => now = observed, + } + } } } diff --git a/backend/core/src/utils/logging.rs b/backend/core/src/utils/logging.rs index 83e487503..4a25e0681 100644 --- a/backend/core/src/utils/logging.rs +++ b/backend/core/src/utils/logging.rs @@ -198,12 +198,6 @@ fn apply_logger_with_context(context: &LoggerContext, config_log_level: Option<& } pub fn init_logger(user_log_level: Option<&str>, config_file: &str) { - // tracing_subscriber::registry() - // .with(console_subscriber::spawn()) // Console layer - // .with(EnvFilter::from_default_env()) - // .with(fmt::layer()) // stdout logging - // .init(); - let context = LoggerContext { cli_log_level: user_log_level.map(std::string::ToString::to_string), env_log_level: std::env::var("TULIPROX_LOG").ok(), diff --git a/backend/core/src/utils/mod.rs b/backend/core/src/utils/mod.rs index 981790d0f..8b281ac23 100644 --- a/backend/core/src/utils/mod.rs +++ b/backend/core/src/utils/mod.rs @@ -18,6 +18,7 @@ mod provider_resolve_token; mod recording_paths; pub mod request_headers; pub mod response_compression; +pub mod runtime_liveness; mod step_measure; mod sys_utils; mod telegram; diff --git a/backend/core/src/utils/runtime_liveness.rs b/backend/core/src/utils/runtime_liveness.rs new file mode 100644 index 000000000..52cb55b0e --- /dev/null +++ b/backend/core/src/utils/runtime_liveness.rs @@ -0,0 +1,527 @@ +//! Scheduler heartbeat and an independent watchdog thread. +//! +//! A wedged Tokio runtime parks every worker on a futex and leaves the I/O +//! driver in `epoll_wait`; at the OS level that is indistinguishable from an +//! idle runtime. No task inside the runtime can report that state, so the +//! detection is split in two: +//! +//! * a heartbeat task records monotonic progress only while the scheduler +//! still runs, and +//! * a plain `std::thread`, outside the runtime, watches the heartbeat age and +//! logs a diagnostic snapshot when progress stops. +//! +//! The watchdog never restarts or interrupts anything. It only reports. + +use crate::model::RuntimeHealth; +use log::{debug, error, info, warn}; +use std::{ + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, OnceLock, + }, + time::{Duration, Instant}, +}; + +pub const DEFAULT_HEARTBEAT_INTERVAL_MS: u64 = 1_000; +pub const DEFAULT_STALL_THRESHOLD_MS: u64 = 10_000; +pub const DEFAULT_STALL_RELOG_MS: u64 = 30_000; +pub const DEFAULT_WATCHDOG_POLL_MS: u64 = 1_000; + +/// How long a confirmed stall must persist before `TULIPROX_WATCHDOG=2` exits +/// the process. The grace gives a transient scheduler pause a chance to clear. +pub const DEFAULT_RESTART_GRACE_MS: u64 = 30_000; + +/// Non-zero so `restart: on-failure` also restarts the container. A zero exit +/// would be treated as a clean shutdown. +const WATCHDOG_RESTART_EXIT_CODE: i32 = 75; + +/// The watchdog only sleeps and reads atomics outside a stall, so it does not +/// need the default thread stack. +const WATCHDOG_STACK_SIZE: usize = 256 * 1024; + +fn monotonic_origin() -> Instant { + static ORIGIN: OnceLock = OnceLock::new(); + *ORIGIN.get_or_init(Instant::now) +} + +/// Monotonic milliseconds since the process first read the liveness clock. +/// +/// Wall-clock reads can jump backwards on NTP adjustments, which would make a +/// heartbeat age meaningless; the watchdog therefore never uses `SystemTime`. +#[must_use] +pub fn monotonic_now_ms() -> u64 { u64::try_from(monotonic_origin().elapsed().as_millis()).unwrap_or(u64::MAX) } + +fn read_env_ms(name: &str, default_ms: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(default_ms) +} + +/// What the watchdog does once a stall is confirmed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum WatchdogMode { + /// No heartbeat task, no watchdog thread. + Off, + /// Report stalls through the log and `/healthcheck.runtime`. + Observe, + /// Like `Observe`, but exit the process after the stall persists so a + /// supervisor can restart it. + Restart, +} + +fn watchdog_mode(value: Option<&str>) -> WatchdogMode { + match value.map(|value| value.trim().to_ascii_lowercase()) { + Some(value) => match value.as_str() { + "2" | "restart" => WatchdogMode::Restart, + "1" | "true" | "on" | "yes" | "enabled" => WatchdogMode::Observe, + _ => WatchdogMode::Off, + }, + None => WatchdogMode::Off, + } +} + +/// The watchdog is opt-in and off by default. `TULIPROX_WATCHDOG` selects the +/// mode: `0`/unset off, `1` observe, `2` observe and restart on a confirmed +/// stall. +fn configured_watchdog_mode() -> WatchdogMode { watchdog_mode(std::env::var("TULIPROX_WATCHDOG").ok().as_deref()) } + +/// Progress shared between the in-runtime heartbeat and the watchdog thread. +/// +/// Every accessor takes the current monotonic instant instead of reading a +/// clock, so the stall decision is testable without threads or time. +pub struct RuntimeLiveness { + started_ms: u64, + heartbeat_interval_ms: u64, + stall_threshold_ms: u64, + restart_on_stall: bool, + last_tick_ms: AtomicU64, + ticks: AtomicU64, + stall_episodes: AtomicU64, + max_schedule_delay_ms: AtomicU64, +} + +impl RuntimeLiveness { + #[must_use] + pub fn new(started_ms: u64, heartbeat_interval_ms: u64, stall_threshold_ms: u64) -> Self { + Self { + started_ms, + heartbeat_interval_ms: heartbeat_interval_ms.max(1), + stall_threshold_ms: stall_threshold_ms.max(1), + restart_on_stall: false, + last_tick_ms: AtomicU64::new(started_ms), + ticks: AtomicU64::new(0), + stall_episodes: AtomicU64::new(0), + max_schedule_delay_ms: AtomicU64::new(0), + } + } + + /// Marks whether the watchdog is allowed to restart the process. + #[must_use] + pub const fn with_restart_on_stall(mut self, restart_on_stall: bool) -> Self { + self.restart_on_stall = restart_on_stall; + self + } + + #[must_use] + pub const fn restart_on_stall(&self) -> bool { self.restart_on_stall } + + #[must_use] + pub const fn heartbeat_interval_ms(&self) -> u64 { self.heartbeat_interval_ms } + + #[must_use] + pub const fn stall_threshold_ms(&self) -> u64 { self.stall_threshold_ms } + + /// Records progress. `schedule_delay_ms` is how late the heartbeat was + /// scheduled relative to its deadline and surfaces scheduler pressure even + /// when the threshold is never crossed. + pub fn tick(&self, now_ms: u64, schedule_delay_ms: u64) { + self.last_tick_ms.store(now_ms, Ordering::Release); + self.ticks.fetch_add(1, Ordering::Relaxed); + self.max_schedule_delay_ms.fetch_max(schedule_delay_ms, Ordering::Relaxed); + } + + /// Saturation covers a heartbeat timestamp that is somehow ahead of `now`. + #[must_use] + pub fn age_ms(&self, now_ms: u64) -> u64 { now_ms.saturating_sub(self.last_tick_ms.load(Ordering::Acquire)) } + + #[must_use] + pub fn is_stalled(&self, now_ms: u64) -> bool { self.age_ms(now_ms) > self.stall_threshold_ms } + + pub fn note_stall(&self) { self.stall_episodes.fetch_add(1, Ordering::Relaxed); } + + #[must_use] + pub fn snapshot(&self, now_ms: u64) -> RuntimeHealth { + let age = self.age_ms(now_ms); + let stalled = age > self.stall_threshold_ms; + RuntimeHealth { + status: if stalled { "stalled".to_string() } else { "alive".to_string() }, + heartbeat_age_ms: age, + heartbeat_interval_ms: self.heartbeat_interval_ms, + stall_threshold_ms: self.stall_threshold_ms, + stalled_for_ms: stalled.then(|| age.saturating_sub(self.stall_threshold_ms)), + ticks: self.ticks.load(Ordering::Relaxed), + stall_episodes: self.stall_episodes.load(Ordering::Relaxed), + max_schedule_delay_ms: self.max_schedule_delay_ms.load(Ordering::Relaxed), + uptime_ms: now_ms.saturating_sub(self.started_ms), + restart_on_stall: self.restart_on_stall, + } + } +} + +static INSTANCE: OnceLock> = OnceLock::new(); + +/// Installs the process-wide liveness state once. Later calls are ignored. +pub fn install(liveness: Arc) -> bool { INSTANCE.set(liveness).is_ok() } + +#[must_use] +pub fn installed() -> Option<&'static Arc> { INSTANCE.get() } + +/// Current liveness snapshot, or `None` when the watchdog is not installed. +#[must_use] +pub fn health_snapshot() -> Option { + INSTANCE.get().map(|liveness| liveness.snapshot(monotonic_now_ms())) +} + +/// Starts the heartbeat task and the watchdog thread, unless already started. +/// +/// `TULIPROX_WATCHDOG` selects the mode: `0`/unset off, `1` observe, `2` observe +/// and restart the process on a confirmed stall. Returns the shared state when +/// started. The interval and threshold can be overridden with +/// `TULIPROX_WATCHDOG_HEARTBEAT_MS` and `TULIPROX_WATCHDOG_STALL_MS`; the relog +/// interval with `TULIPROX_WATCHDOG_RELOG_MS`; the restart grace with +/// `TULIPROX_WATCHDOG_RESTART_GRACE_MS`. +pub fn start(runtime: &tokio::runtime::Handle) -> Option> { + let mode = configured_watchdog_mode(); + if mode == WatchdogMode::Off { + debug!("Runtime liveness watchdog disabled (set TULIPROX_WATCHDOG=1 to observe, =2 to also restart)"); + return None; + } + + let heartbeat_interval_ms = read_env_ms("TULIPROX_WATCHDOG_HEARTBEAT_MS", DEFAULT_HEARTBEAT_INTERVAL_MS); + let stall_threshold_ms = read_env_ms("TULIPROX_WATCHDOG_STALL_MS", DEFAULT_STALL_THRESHOLD_MS); + let relog_ms = read_env_ms("TULIPROX_WATCHDOG_RELOG_MS", DEFAULT_STALL_RELOG_MS); + let restart_grace_ms = read_env_ms("TULIPROX_WATCHDOG_RESTART_GRACE_MS", DEFAULT_RESTART_GRACE_MS); + + let liveness = Arc::new( + RuntimeLiveness::new(monotonic_now_ms(), heartbeat_interval_ms, stall_threshold_ms) + .with_restart_on_stall(mode == WatchdogMode::Restart), + ); + if !install(Arc::clone(&liveness)) { + return None; + } + + spawn_heartbeat(Arc::clone(&liveness)); + match spawn_watchdog(Arc::clone(&liveness), runtime.clone(), relog_ms, restart_grace_ms) { + Ok(_) => info!( + "Runtime liveness watchdog started: mode={mode:?}, heartbeat {heartbeat_interval_ms} ms, stall threshold {stall_threshold_ms} ms" + ), + Err(err) => warn!("Failed to start runtime liveness watchdog thread: {err}"), + } + Some(liveness) +} + +/// Spawns the in-runtime heartbeat task. +fn spawn_heartbeat(liveness: Arc) { + let interval = Duration::from_millis(liveness.heartbeat_interval_ms()); + tokio::spawn(async move { + let mut expected = Instant::now() + interval; + loop { + tokio::time::sleep_until(tokio::time::Instant::from_std(expected)).await; + let now = Instant::now(); + let delay_ms = u64::try_from(now.saturating_duration_since(expected).as_millis()).unwrap_or(u64::MAX); + liveness.tick(monotonic_now_ms(), delay_ms); + expected += interval; + // After a long stall, sleep_until would return immediately until it + // catches up; resync so recovery does not become a tight loop. + if expected < now { + expected = now + interval; + } + } + }); +} + +/// Spawns the watchdog thread that lives outside the Tokio runtime. +fn spawn_watchdog( + liveness: Arc, + runtime: tokio::runtime::Handle, + relog_ms: u64, + restart_grace_ms: u64, +) -> std::io::Result> { + std::thread::Builder::new().name("tuliprox-watchdog".to_string()).stack_size(WATCHDOG_STACK_SIZE).spawn(move || { + run_watchdog(&liveness, &runtime, relog_ms, restart_grace_ms); + }) +} + +/// The decision the watchdog makes on each poll. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum WatchdogVerdict { + /// Within the threshold and not in a stall, or still inside an episode but + /// before the relog interval is due. + Quiet, + /// Above the threshold and it is time to (re)log the stall. + LogStall, + /// The heartbeat is fresh again after an episode. + Recovered, +} + +fn watchdog_verdict( + age_ms: u64, + stall_threshold_ms: u64, + in_stall: bool, + since_last_log_ms: u64, + relog_ms: u64, +) -> WatchdogVerdict { + if age_ms > stall_threshold_ms { + if in_stall && since_last_log_ms < relog_ms { + WatchdogVerdict::Quiet + } else { + WatchdogVerdict::LogStall + } + } else if in_stall { + WatchdogVerdict::Recovered + } else { + WatchdogVerdict::Quiet + } +} + +/// Whether `TULIPROX_WATCHDOG=2` should exit the process now. +fn restart_due(restart_on_stall: bool, stall_started_ms: Option, now_ms: u64, grace_ms: u64) -> bool { + restart_on_stall && stall_started_ms.is_some_and(|started| now_ms.saturating_sub(started) >= grace_ms) +} + +fn run_watchdog(liveness: &RuntimeLiveness, runtime: &tokio::runtime::Handle, relog_ms: u64, restart_grace_ms: u64) { + let poll = Duration::from_millis(DEFAULT_WATCHDOG_POLL_MS); + let threshold_ms = liveness.stall_threshold_ms(); + let restart_on_stall = liveness.restart_on_stall(); + let mut in_stall = false; + let mut last_log_ms = 0_u64; + let mut stall_started_ms: Option = None; + + loop { + std::thread::sleep(poll); + let now = monotonic_now_ms(); + let age = liveness.age_ms(now); + let since_last_log = now.saturating_sub(last_log_ms); + match watchdog_verdict(age, threshold_ms, in_stall, since_last_log, relog_ms) { + WatchdogVerdict::Quiet => {} + WatchdogVerdict::LogStall => { + if !in_stall { + liveness.note_stall(); + stall_started_ms = Some(now); + } + let snapshot = liveness.snapshot(now); + warn!( + "Runtime liveness stall: heartbeat silent for {age} ms (threshold {threshold_ms} ms, episode {})", + snapshot.stall_episodes + ); + log_runtime_diagnostics(&snapshot, runtime, age); + last_log_ms = now; + in_stall = true; + } + WatchdogVerdict::Recovered => { + info!("Runtime liveness recovered: heartbeat resumed after {age} ms of silence"); + in_stall = false; + stall_started_ms = None; + } + } + + if restart_due(restart_on_stall, stall_started_ms, now, restart_grace_ms) { + error!( + "Runtime liveness restart: heartbeat silent for {age} ms (threshold {threshold_ms} ms) and the stall \ + persisted past the {restart_grace_ms} ms grace; exiting with code {WATCHDOG_RESTART_EXIT_CODE} so the \ + supervisor can restart the process" + ); + std::process::exit(WATCHDOG_RESTART_EXIT_CODE); + } + } +} + +fn log_runtime_diagnostics(snapshot: &RuntimeHealth, runtime: &tokio::runtime::Handle, age_ms: u64) { + let metrics = runtime.metrics(); + warn!( + "Runtime diagnostics: heartbeat_age_ms={age_ms}, heartbeat_interval_ms={}, stall_threshold_ms={}, ticks={}, max_schedule_delay_ms={}, uptime_ms={}, workers={}, alive_tasks={}, global_queue_depth={}", + snapshot.heartbeat_interval_ms, + snapshot.stall_threshold_ms, + snapshot.ticks, + snapshot.max_schedule_delay_ms, + snapshot.uptime_ms, + metrics.num_workers(), + metrics.num_alive_tasks(), + metrics.global_queue_depth(), + ); + + #[cfg(target_has_atomic = "64")] + log_worker_diagnostics(&metrics); + + #[cfg(target_os = "linux")] + log_thread_inventory(); +} + +// Only metrics that are stable without `tokio_unstable` may be used here: the +// same source must compile in the default build. `worker_poll_count`, +// `worker_local_queue_depth` and the blocking-pool counters are behind +// `tokio_unstable` and are deliberately omitted. +#[cfg(target_has_atomic = "64")] +fn log_worker_diagnostics(metrics: &tokio::runtime::RuntimeMetrics) { + for worker in 0..metrics.num_workers() { + warn!( + "Runtime worker {worker}: park_count={}, total_busy={:?}", + metrics.worker_park_count(worker), + metrics.worker_total_busy_duration(worker), + ); + } +} + +/// Logs what every thread in this process is doing, mirroring the manual +/// `/proc//task/*/stack` inspection used to diagnose a hang. +#[cfg(target_os = "linux")] +fn log_thread_inventory() { + let Ok(entries) = std::fs::read_dir("/proc/self/task") else { return }; + + let mut threads: Vec = Vec::new(); + for entry in entries.flatten() { + let Some(tid) = entry.file_name().into_string().ok() else { continue }; + let base = entry.path(); + let comm = + std::fs::read_to_string(base.join("comm")).map_or_else(|_| "?".to_string(), |name| name.trim().to_string()); + let state = read_proc_state(&base.join("stat")); + let wchan = std::fs::read_to_string(base.join("wchan")) + .map_or_else(|_| "?".to_string(), |value| value.trim().to_string()); + threads.push(format!("tid={tid} name={comm} state={state} wchan={wchan}")); + } + threads.sort(); + warn!("Runtime thread inventory ({} threads):\n{}", threads.len(), threads.join("\n")); +} + +/// Reads the state field from `/proc//stat`. +/// +/// The command name in the second field is wrapped in parentheses and may +/// itself contain spaces and parentheses, so the state is the token following +/// the *last* `)`. +#[cfg(target_os = "linux")] +fn read_proc_state(stat_path: &std::path::Path) -> String { + let Ok(stat) = std::fs::read_to_string(stat_path) else { return "?".to_string() }; + stat.rfind(')') + .and_then(|closing| stat[closing + 1..].split_whitespace().next()) + .map_or_else(|| "?".to_string(), ToString::to_string) +} + +#[cfg(test)] +mod tests { + use super::{ + monotonic_now_ms, restart_due, spawn_heartbeat, watchdog_mode, watchdog_verdict, RuntimeLiveness, WatchdogMode, + WatchdogVerdict, + }; + use std::{ + sync::Arc, + time::{Duration, Instant}, + }; + + #[test] + fn watchdog_mode_is_off_by_default_and_maps_the_supported_values() { + assert_eq!(watchdog_mode(None), WatchdogMode::Off); + for value in ["0", "false", "FALSE", " off ", "no", "disabled", "", "9"] { + assert_eq!(watchdog_mode(Some(value)), WatchdogMode::Off, "expected {value:?} to be off"); + } + for value in ["1", "true", "TRUE", " on ", "yes", "enabled"] { + assert_eq!(watchdog_mode(Some(value)), WatchdogMode::Observe, "expected {value:?} to observe"); + } + for value in ["2", "RESTART", " restart "] { + assert_eq!(watchdog_mode(Some(value)), WatchdogMode::Restart, "expected {value:?} to restart"); + } + } + + #[test] + fn restart_is_due_only_in_restart_mode_after_the_grace() { + assert!(!restart_due(false, Some(1_000), 100_000, 30_000)); + assert!(!restart_due(true, None, 100_000, 30_000)); + assert!(!restart_due(true, Some(80_000), 100_000, 30_000)); + assert!(restart_due(true, Some(70_000), 100_000, 30_000)); + assert!(restart_due(true, Some(70_000), 100_000, 0)); + } + + #[test] + fn restart_arm_is_reported_in_the_snapshot() { + let liveness = RuntimeLiveness::new(0, 1_000, 5_000).with_restart_on_stall(true); + assert!(liveness.restart_on_stall()); + assert!(liveness.snapshot(0).restart_on_stall); + assert!(!RuntimeLiveness::new(0, 1_000, 5_000).snapshot(0).restart_on_stall); + } + + #[tokio::test] + async fn heartbeat_task_records_progress() { + let liveness = Arc::new(RuntimeLiveness::new(monotonic_now_ms(), 5, 5_000)); + spawn_heartbeat(Arc::clone(&liveness)); + let deadline = Instant::now() + Duration::from_secs(5); + while liveness.snapshot(monotonic_now_ms()).ticks == 0 { + assert!(Instant::now() < deadline, "heartbeat task never ticked"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!liveness.is_stalled(monotonic_now_ms())); + } + + #[test] + fn fresh_heartbeat_reports_alive() { + let liveness = RuntimeLiveness::new(1_000, 1_000, 5_000); + liveness.tick(2_000, 3); + let snapshot = liveness.snapshot(2_500); + assert_eq!(snapshot.status, "alive"); + assert_eq!(snapshot.heartbeat_age_ms, 500); + assert_eq!(snapshot.stalled_for_ms, None); + assert_eq!(snapshot.ticks, 1); + assert_eq!(snapshot.max_schedule_delay_ms, 3); + assert_eq!(snapshot.uptime_ms, 1_500); + } + + #[test] + fn silent_heartbeat_reports_stalled_with_overflow_age() { + let liveness = RuntimeLiveness::new(0, 1_000, 5_000); + liveness.tick(1_000, 0); + let snapshot = liveness.snapshot(9_000); + assert_eq!(snapshot.status, "stalled"); + assert_eq!(snapshot.heartbeat_age_ms, 8_000); + assert_eq!(snapshot.stalled_for_ms, Some(3_000)); + assert!(liveness.is_stalled(9_000)); + assert!(!liveness.is_stalled(5_500)); + } + + #[test] + fn heartbeat_age_saturates_when_clock_moves_backwards() { + let liveness = RuntimeLiveness::new(10_000, 1_000, 5_000); + assert_eq!(liveness.age_ms(1_000), 0); + } + + #[test] + fn max_schedule_delay_keeps_the_worst_observation() { + let liveness = RuntimeLiveness::new(0, 1_000, 5_000); + liveness.tick(1_000, 10); + liveness.tick(2_000, 4); + liveness.tick(3_000, 25); + assert_eq!(liveness.snapshot(3_000).max_schedule_delay_ms, 25); + } + + #[test] + fn stall_episodes_are_counted_once_per_episode() { + let liveness = RuntimeLiveness::new(0, 1_000, 5_000); + assert_eq!(liveness.snapshot(0).stall_episodes, 0); + liveness.note_stall(); + liveness.note_stall(); + assert_eq!(liveness.snapshot(0).stall_episodes, 2); + } + + #[test] + fn verdict_logs_on_entry_then_respects_relog_interval() { + assert_eq!(watchdog_verdict(1_000, 5_000, false, 0, 30_000), WatchdogVerdict::Quiet); + assert_eq!(watchdog_verdict(6_000, 5_000, false, 0, 30_000), WatchdogVerdict::LogStall); + assert_eq!(watchdog_verdict(6_000, 5_000, true, 1_000, 30_000), WatchdogVerdict::Quiet); + assert_eq!(watchdog_verdict(6_000, 5_000, true, 30_000, 30_000), WatchdogVerdict::LogStall); + } + + #[test] + fn verdict_reports_recovery_only_after_an_episode() { + assert_eq!(watchdog_verdict(100, 5_000, false, 0, 30_000), WatchdogVerdict::Quiet); + assert_eq!(watchdog_verdict(100, 5_000, true, 0, 30_000), WatchdogVerdict::Recovered); + } +} diff --git a/backend/dvr/Cargo.toml b/backend/dvr/Cargo.toml index f1a3c9745..fcd484b26 100644 --- a/backend/dvr/Cargo.toml +++ b/backend/dvr/Cargo.toml @@ -26,6 +26,8 @@ thiserror.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "fs", "time", "macros", "sync", "process"] } tokio-util = { version = "0.7.19", features = ["io", "io-util"] } url.workspace = true + +[target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61.2", features = ["Win32_Storage_FileSystem"] } [dev-dependencies] diff --git a/backend/hls/src/hls_ctx.rs b/backend/hls/src/hls_ctx.rs index 0a080d4f8..1da94670c 100644 --- a/backend/hls/src/hls_ctx.rs +++ b/backend/hls/src/hls_ctx.rs @@ -115,7 +115,7 @@ impl HlsCtx { let event_manager = Arc::new(tuliprox_session::EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(&app_config, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + active_provider.set_shared_stream_manager(&shared_stream_manager); let loaded = app_config.config.load(); let geoip = Arc::new(ArcSwapOption::default()); diff --git a/backend/hls/src/lib.rs b/backend/hls/src/lib.rs index dbd4b1f80..c752c372a 100644 --- a/backend/hls/src/lib.rs +++ b/backend/hls/src/lib.rs @@ -256,11 +256,11 @@ pub(crate) use self::{ HlsRecoveryTriggerSource, }, origin::{ - begin_hls_origin_account_io, begin_hls_origin_account_io_bounded, classify_account_binding_protection, - finish_hls_origin_account_io, HlsAccountBindingProtection, HlsAccountOverlapTiming, - HlsBoundAccountAcquireErrorKind, HlsEffectiveOriginAcquirePolicy, HlsEffectiveOriginAcquirePolicyState, - HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginAccountIoLeaseGuard, HlsOriginAccountRebindState, - HlsOriginIoContext, HlsOriginSource, HlsOriginWorkClass, + begin_hls_origin_account_io_bounded, classify_account_binding_protection, finish_hls_origin_account_io, + HlsAccountBindingProtection, HlsAccountOverlapTiming, HlsBoundAccountAcquireErrorKind, + HlsEffectiveOriginAcquirePolicy, HlsEffectiveOriginAcquirePolicyState, HlsOriginAccountBinding, + HlsOriginAccountIoLease, HlsOriginAccountIoLeaseGuard, HlsOriginAccountRebindState, HlsOriginIoContext, + HlsOriginSource, HlsOriginWorkClass, }, origin_progress::HlsOriginProgressDecision, paths::{HlsMapFile, HlsSegmentFile, TransientResourceFile}, diff --git a/backend/hls/src/manager.rs b/backend/hls/src/manager.rs index 21bcd5c1c..8d1b9d3d9 100644 --- a/backend/hls/src/manager.rs +++ b/backend/hls/src/manager.rs @@ -184,6 +184,7 @@ pub enum HlsMediaActivityCommitOutcome { #[derive(Debug, Clone, Copy)] enum HlsMediaActivityCommitKind { Access, + Delivered, LiveSegmentCompletion(HlsPlaybackRequestToken), } @@ -2327,6 +2328,25 @@ impl HlsProxyManager { .await } + pub async fn mark_delivered_media_for_lease( + &self, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + now_ms: u64, + ) -> HlsMediaActivityCommitOutcome { + self.commit_media_activity_if_identity_matches( + session, + lease_id, + proxy_session_id, + lease_identity, + now_ms, + HlsMediaActivityCommitKind::Delivered, + ) + .await + } + async fn commit_media_activity_if_identity_matches( &self, session: &HlsSessionHandle, @@ -2468,7 +2488,7 @@ impl HlsProxyManager { }; } let (current, capacity_protection_released) = match kind { - HlsMediaActivityCommitKind::Access => { + HlsMediaActivityCommitKind::Access | HlsMediaActivityCommitKind::Delivered => { (leases.media_identity_is_current(lease_id, proxy_session_id, lease_identity, now_ms), false) } HlsMediaActivityCommitKind::LiveSegmentCompletion(token) => { @@ -2489,6 +2509,9 @@ impl HlsProxyManager { }; } session.mark_authorized_media_access(now_ms); + if lease_identity.is_live() && matches!(kind, HlsMediaActivityCommitKind::Delivered) { + session.activity.last_delivered_media_at_ms = Some(now_ms); + } HlsMediaActivityCommitAttempt::Completed { outcome: HlsMediaActivityCommitOutcome::Committed, evidence_changed: capacity_protection_released, @@ -2635,7 +2658,8 @@ impl HlsProxyManager { maps = maps.saturating_add(session.maps.len()); transient_resources = transient_resources.saturating_add(session.transient.resources.len()); transient_objects = transient_objects.saturating_add(session.transient.object_cache.len()); - active_origin_work = active_origin_work.saturating_add(session.activity.active_origin_work_count); + active_origin_work = active_origin_work + .saturating_add(session.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire)); active_segment_fetches = active_segment_fetches.saturating_add(session.active_segment_fetches); active_map_fetches = active_map_fetches.saturating_add(session.active_map_fetches); } diff --git a/backend/hls/src/origin.rs b/backend/hls/src/origin.rs index cfdfbcf06..2d8b270c9 100644 --- a/backend/hls/src/origin.rs +++ b/backend/hls/src/origin.rs @@ -7,12 +7,11 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; -use tokio::{ - sync::{Mutex, Notify}, - time::timeout, +use tokio::{sync::Notify, time::timeout}; +use tuliprox_core::model::{ + is_input_expired, PlaybackKind, PlaybackRequestId, ProviderAllocation, ProviderBindingTag, ProviderHandle, }; -use tuliprox_core::model::{is_input_expired, ProviderAllocation, ProviderHandle}; -use tuliprox_session::ConnectionKind; +use tuliprox_session::{CleanupEvent, ConnectionKind, PlaybackLeaseRef}; const HLS_ACCOUNT_OVERLAP_FALLBACK_TARGET_DURATION_MS: u64 = 15_000; const HLS_ORIGIN_ACCOUNT_IO_WAIT_RECHECK: Duration = Duration::from_millis(25); @@ -102,19 +101,26 @@ pub struct HlsOriginAccountBinding { pub input_name: Arc, pub account_name: Arc, pub session_owner: String, + pub playback_request_id: Option, pub pinned_at_ms: u64, pub last_origin_io_at_ms: Option, pub last_reservation_refresh_at_ms: Option, pub binding_mode: HlsOriginAccountBindingMode, pub generation: u64, + /// The provider lease's binding tag captured when this binding acquired its slot. + /// A stale detach/clear carries the old tag and cannot delete a successor lease + /// on the same account (Aβ†’A rebind or remove/recreate). + pub provider_binding_tag: Option, } pub struct HlsOriginAccountIoLease { pub account_name: Arc, pub session_owner: String, - pub active_io_count: usize, - provider_handle: Option, - acquiring: bool, + pub generation: u64, + pub active_io_count: std::sync::atomic::AtomicUsize, + provider_handle: std::sync::Mutex>, + cleanup_permit: std::sync::Mutex>>, + acquiring: std::sync::atomic::AtomicBool, notify: Arc, } @@ -123,18 +129,73 @@ impl HlsOriginAccountIoLease { Self { account_name: Arc::clone(&binding.account_name), session_owner: binding.session_owner.clone(), - active_io_count: 0, - provider_handle: None, - acquiring: true, + generation: binding.generation, + active_io_count: std::sync::atomic::AtomicUsize::new(0), + provider_handle: std::sync::Mutex::new(None), + cleanup_permit: std::sync::Mutex::new(None), + acquiring: std::sync::atomic::AtomicBool::new(true), + notify: Arc::new(Notify::new()), + } + } + + #[cfg(test)] + pub fn active_for_test(binding: &HlsOriginAccountBinding, active_io_count: usize) -> Self { + Self { + account_name: Arc::clone(&binding.account_name), + session_owner: binding.session_owner.clone(), + generation: binding.generation, + active_io_count: std::sync::atomic::AtomicUsize::new(active_io_count), + provider_handle: std::sync::Mutex::new(None), + cleanup_permit: std::sync::Mutex::new(None), + acquiring: std::sync::atomic::AtomicBool::new(false), notify: Arc::new(Notify::new()), } } fn matches_binding(&self, binding: &HlsOriginAccountBinding) -> bool { - self.account_name == binding.account_name && self.session_owner == binding.session_owner + self.account_name == binding.account_name + && self.session_owner == binding.session_owner + && self.generation == binding.generation } - pub const fn is_active_or_acquiring(&self) -> bool { self.active_io_count > 0 || self.acquiring } + pub fn is_active_or_acquiring(&self) -> bool { + self.active_io_count.load(std::sync::atomic::Ordering::Acquire) > 0 + || self.acquiring.load(std::sync::atomic::Ordering::Acquire) + } + + fn try_join(&self) -> bool { + if self.acquiring.load(std::sync::atomic::Ordering::Acquire) { + return false; + } + let mut count = self.active_io_count.load(std::sync::atomic::Ordering::Acquire); + loop { + if count == 0 { + return false; + } + let Some(next) = count.checked_add(1) else { return false }; + match self.active_io_count.compare_exchange_weak( + count, + next, + std::sync::atomic::Ordering::AcqRel, + std::sync::atomic::Ordering::Acquire, + ) { + Ok(_) => return true, + Err(observed) => count = observed, + } + } + } + + fn take_resource(&self) -> Option<(ProviderHandle, tokio::sync::mpsc::OwnedPermit)> { + let handle = match self.provider_handle.lock() { + Ok(mut guard) => guard.take(), + Err(poisoned) => poisoned.into_inner().take(), + }?; + let permit = match self.cleanup_permit.lock() { + Ok(mut guard) => guard.take(), + Err(poisoned) => poisoned.into_inner().take(), + }?; + Some((handle, permit)) + } } impl fmt::Debug for HlsOriginAccountIoLease { @@ -142,23 +203,100 @@ impl fmt::Debug for HlsOriginAccountIoLease { f.debug_struct("HlsOriginAccountIoLease") .field("account_name", &self.account_name) .field("session_owner", &"") - .field("active_io_count", &self.active_io_count) - .field("has_provider_handle", &self.provider_handle.is_some()) - .field("acquiring", &self.acquiring) + .field("generation", &self.generation) + .field("active_io_count", &self.active_io_count.load(std::sync::atomic::Ordering::Relaxed)) + .field("has_provider_handle", &self.provider_handle.lock().is_ok_and(|guard| guard.is_some())) + .field("acquiring", &self.acquiring.load(std::sync::atomic::Ordering::Relaxed)) .finish_non_exhaustive() } } -#[derive(Debug, Clone)] pub struct HlsOriginAccountIoLeaseGuard { binding: HlsOriginAccountBinding, + lease: Arc, + disarmed: bool, +} + +impl fmt::Debug for HlsOriginAccountIoLeaseGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HlsOriginAccountIoLeaseGuard") + .field("binding", &self.binding) + .field("disarmed", &self.disarmed) + .finish_non_exhaustive() + } } impl HlsOriginAccountIoLeaseGuard { + pub fn new(binding: HlsOriginAccountBinding, lease: Arc) -> Self { + Self { binding, lease, disarmed: false } + } + pub fn binding(&self) -> &HlsOriginAccountBinding { &self.binding } + + pub fn disarm(&mut self) { self.disarmed = true; } +} + +impl Drop for HlsOriginAccountIoLeaseGuard { + fn drop(&mut self) { + if self.disarmed { + return; + } + self.disarmed = true; + if self.lease.active_io_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel) == 1 { + if let Some((handle, permit)) = self.lease.take_resource() { + permit.send(CleanupEvent::ReleaseProviderHandle { handle: Some(handle) }); + } + } + } +} + +struct ManagedHandleGuard { + handle: Option, + cleanup_permit: Option>, +} + +impl ManagedHandleGuard { + fn new(handle: ProviderHandle, cleanup_permit: tokio::sync::mpsc::OwnedPermit) -> Self { + Self { handle: Some(handle), cleanup_permit: Some(cleanup_permit) } + } + + fn take_resource(&mut self) -> Option<(ProviderHandle, tokio::sync::mpsc::OwnedPermit)> { + Some((self.handle.take()?, self.cleanup_permit.take()?)) + } +} + +impl Drop for ManagedHandleGuard { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + if let Some(permit) = self.cleanup_permit.take() { + permit.send(CleanupEvent::ReleaseProviderHandle { handle: Some(handle) }); + } + } + } +} + +pub struct PendingAcquireGuard { + lease: Arc, + disarmed: bool, +} + +impl PendingAcquireGuard { + fn new(lease: Arc) -> Self { Self { lease, disarmed: false } } + + fn disarm(&mut self) { self.disarmed = true; } +} + +impl Drop for PendingAcquireGuard { + fn drop(&mut self) { + if self.disarmed { + return; + } + self.disarmed = true; + self.lease.acquiring.store(false, std::sync::atomic::Ordering::Release); + self.lease.notify.notify_waiters(); + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct HlsAccountOverlapTiming { pub target_duration_ms: u64, pub hard_active_window_ms: u64, @@ -203,7 +341,7 @@ impl HlsAccountBindingProtection { pub fn classify_account_binding_protection( last_authorized_media_at_ms: Option, now_ms: u64, - timing: HlsAccountOverlapTiming, + timing: &HlsAccountOverlapTiming, ) -> HlsAccountBindingProtection { let Some(last_media) = last_authorized_media_at_ms else { return HlsAccountBindingProtection::NoMediaYet; @@ -258,11 +396,13 @@ impl HlsOriginAccountBinding { input_name, account_name, session_owner: build_hls_origin_session_owner(proxy_session_id), + playback_request_id: None, pinned_at_ms: now_ms, last_origin_io_at_ms: None, last_reservation_refresh_at_ms: None, binding_mode: HlsOriginAccountBindingMode::Active, generation: 0, + provider_binding_tag: None, } } @@ -277,11 +417,13 @@ impl HlsOriginAccountBinding { input_name, account_name, session_owner, + playback_request_id: None, pinned_at_ms: now_ms, last_origin_io_at_ms: None, last_reservation_refresh_at_ms: None, binding_mode: HlsOriginAccountBindingMode::Active, generation, + provider_binding_tag: None, } } @@ -324,11 +466,13 @@ impl fmt::Debug for HlsOriginAccountBinding { .field("input_name", &self.input_name) .field("account_name", &self.account_name) .field("session_owner", &"") + .field("playback_request_id", &self.playback_request_id) .field("pinned_at_ms", &self.pinned_at_ms) .field("last_origin_io_at_ms", &self.last_origin_io_at_ms) .field("last_reservation_refresh_at_ms", &self.last_reservation_refresh_at_ms) .field("binding_mode", &self.binding_mode) .field("generation", &self.generation) + .field("provider_binding_tag", &self.provider_binding_tag) .finish() } } @@ -480,7 +624,7 @@ pub struct HlsOriginIoContext { pub priority: i8, pub connection_kind: ConnectionKind, pub reservation_ttl_secs: u64, - pub preacquired_provider_handle: Option>>>, + pub preacquired_provider_handle: Option>>>, pub started_generation: Option, } @@ -491,13 +635,19 @@ impl HlsOriginIoContext { } pub fn with_preacquired_provider_handle(mut self, provider_handle: ProviderHandle) -> Self { - self.preacquired_provider_handle = Some(Arc::new(Mutex::new(Some(provider_handle)))); + let managed = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&self.ctx.active_provider), provider_handle); + self.preacquired_provider_handle = Some(Arc::new(std::sync::Mutex::new(Some(managed)))); self } - pub async fn take_preacquired_provider_handle(&self) -> Option { + pub fn take_preacquired_provider_handle(&self) -> Option { let handle = self.preacquired_provider_handle.as_ref()?; - handle.lock().await.take() + let mut guard = match handle.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + guard.take() } } @@ -518,7 +668,7 @@ pub fn hls_origin_account_status(ctx: &HlsCtx, binding: &HlsOriginAccountBinding HlsOriginAccountStatus::Known } -pub async fn acquire_bound_hls_origin_account_handle( +pub fn acquire_bound_hls_origin_account_handle( ctx: &HlsCtx, binding: &HlsOriginAccountBinding, client_addr: &SocketAddr, @@ -534,28 +684,21 @@ pub async fn acquire_bound_hls_origin_account_handle( HlsOriginAccountStatus::Expired => return Err(HlsBoundAccountAcquireErrorKind::Expired), HlsOriginAccountStatus::Known => {} } - if ctx - .active_provider - .is_provider_reserved_for_other_session(&binding.account_name, Some(&binding.session_owner)) - .await - { + if ctx.active_provider.is_provider_reserved_for_other_session(&binding.account_name, Some(&binding.session_owner)) { return Err(HlsBoundAccountAcquireErrorKind::ReservedForOther); } - let handle = ctx - .active_provider - .acquire_exact_connection_with_grace_for_session( - &binding.account_name, - client_addr, - allow_grace, - priority, - connection_kind, - Some(&binding.session_owner), - ) - .await; + let handle = ctx.active_provider.acquire_exact_connection_with_lease_for_session( + &binding.account_name, + client_addr, + allow_grace, + priority, + connection_kind, + Some(PlaybackLeaseRef::new(&binding.session_owner, PlaybackKind::LiveHls)), + ); if let Some(handle) = handle { return Ok(handle); } - let kind = if ctx.active_provider.is_exhausted(&binding.account_name).await { + let kind = if ctx.active_provider.is_exhausted(&binding.account_name) { HlsBoundAccountAcquireErrorKind::Exhausted } else { HlsBoundAccountAcquireErrorKind::Unavailable @@ -595,65 +738,108 @@ async fn begin_hls_origin_account_io_inner( return Err(HlsBoundAccountAcquireErrorKind::Detached); } - if matches!( - reserve_hls_origin_account_io_slot(session, binding, acquire_timeout).await?, - HlsOriginAccountIoSlot::Joined - ) { - if let Some(unused_handle) = origin_io.take_preacquired_provider_handle().await { - origin_io.ctx.connection_manager.release_provider_handle(Some(unused_handle)).await; - } - return Ok(HlsOriginAccountIoLeaseGuard { binding: binding.clone() }); + let deadline = acquire_timeout.map(|duration| Instant::now() + duration); + let slot = reserve_hls_origin_account_io_slot(session, binding, deadline).await?; + if let HlsOriginAccountIoSlot::Joined(lease) = slot { + let guard = HlsOriginAccountIoLeaseGuard::new(binding.clone(), lease); + // Release any unused pre-acquired handle synchronously on drop. It needs no + // cleanup permit, so a join cannot self-block on a single-permit cleanup queue. + drop(origin_io.take_preacquired_provider_handle()); + return Ok(guard); } - let acquired_handle = if let Some(handle) = origin_io.take_preacquired_provider_handle().await { - Ok(handle) + let HlsOriginAccountIoSlot::Acquire(lease) = slot else { + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + }; + // Own the pending lease before the first suspend after the slot reserve: the + // guard's Drop clears `acquiring` and wakes waiters if the cleanup-permit wait + // below times out, is cancelled, or the cleanup channel is closed. + let mut pending_guard = PendingAcquireGuard::new(Arc::clone(&lease)); + let cleanup_permit = reserve_cleanup_permit(origin_io, deadline).await?; + + let acquired_handle = if let Some(mut managed) = origin_io.take_preacquired_provider_handle() { + // Disarm the managed owner only after the cleanup permit is already reserved, + // so the naked handle is immediately paired with its release responsibility. + // The wrapper always carries its handle; fall back to a fresh acquire defensively. + match managed.disarm() { + Some(handle) => Ok(handle), + None => acquire_bound_hls_origin_account_handle( + &origin_io.ctx, + binding, + &origin_io.client_addr, + origin_io.allow_grace, + origin_io.priority, + origin_io.connection_kind, + ), + } } else { - let acquire = acquire_bound_hls_origin_account_handle( + acquire_bound_hls_origin_account_handle( &origin_io.ctx, binding, &origin_io.client_addr, origin_io.allow_grace, origin_io.priority, origin_io.connection_kind, - ); - if let Some(acquire_timeout) = acquire_timeout { - if let Ok(result) = timeout(acquire_timeout, acquire).await { - result - } else { - clear_pending_hls_origin_account_io_lease(session, binding).await; - return Err(HlsBoundAccountAcquireErrorKind::AcquireTimedOut); - } - } else { - acquire.await - } + ) }; match acquired_handle { - Ok(handle) => store_acquired_hls_origin_account_io_handle(origin_io, session, binding, handle).await, + Ok(handle) => { + store_acquired_hls_origin_account_io_handle( + origin_io, + session, + binding, + handle, + cleanup_permit, + pending_guard, + deadline, + ) + .await + } Err(err) => { - clear_pending_hls_origin_account_io_lease(session, binding).await; + clear_pending_hls_origin_account_io_lease(session, &pending_guard.lease, deadline).await; + pending_guard.disarm(); Err(err) } } } +async fn reserve_cleanup_permit( + origin_io: &HlsOriginIoContext, + deadline: Option, +) -> Result, HlsBoundAccountAcquireErrorKind> { + let reserve = origin_io.ctx.connection_manager.control_cleanup_tx().reserve_owned(); + match deadline { + Some(deadline) => timeout(deadline.saturating_duration_since(Instant::now()), reserve) + .await + .map_err(|_| HlsBoundAccountAcquireErrorKind::AcquireTimedOut)? + .map_err(|_| HlsBoundAccountAcquireErrorKind::Unavailable), + None => reserve.await.map_err(|_| HlsBoundAccountAcquireErrorKind::Unavailable), + } +} + async fn reserve_hls_origin_account_io_slot( session: &HlsSessionHandle, binding: &HlsOriginAccountBinding, - acquire_timeout: Option, + wait_deadline: Option, ) -> Result { - let wait_deadline = acquire_timeout.map(|duration| Instant::now() + duration); loop { - let outcome = { + let reserve = async { let mut session = session.write().await; try_reserve_hls_origin_account_io_slot(&mut session, binding) }; + let outcome = match wait_deadline { + Some(deadline) => timeout(deadline.saturating_duration_since(Instant::now()), reserve) + .await + .map_err(|_| HlsBoundAccountAcquireErrorKind::WaitTimedOut)?, + None => reserve.await, + }; match outcome { - HlsOriginAccountIoReserveOutcome::Joined => { - return Ok(HlsOriginAccountIoSlot::Joined); + HlsOriginAccountIoReserveOutcome::Joined(lease) => { + return Ok(HlsOriginAccountIoSlot::Joined(lease)); } - HlsOriginAccountIoReserveOutcome::Acquire => { - return Ok(HlsOriginAccountIoSlot::Acquire); + HlsOriginAccountIoReserveOutcome::Acquire(lease) => { + return Ok(HlsOriginAccountIoSlot::Acquire(lease)); } HlsOriginAccountIoReserveOutcome::Wait(notify) => { if let Some(deadline) = wait_deadline { @@ -673,24 +859,24 @@ async fn reserve_hls_origin_account_io_slot( } } } - HlsOriginAccountIoReserveOutcome::Unavailable => { - return Err(HlsBoundAccountAcquireErrorKind::Unavailable); - } HlsOriginAccountIoReserveOutcome::ReservedForOther => { return Err(HlsBoundAccountAcquireErrorKind::ReservedForOther); } + HlsOriginAccountIoReserveOutcome::Unavailable => { + return Err(HlsBoundAccountAcquireErrorKind::Unavailable); + } } } } enum HlsOriginAccountIoSlot { - Joined, - Acquire, + Joined(Arc), + Acquire(Arc), } enum HlsOriginAccountIoReserveOutcome { - Joined, - Acquire, + Joined(Arc), + Acquire(Arc), Wait(Arc), ReservedForOther, Unavailable, @@ -704,22 +890,23 @@ fn try_reserve_hls_origin_account_io_slot( return HlsOriginAccountIoReserveOutcome::Unavailable; } - match session.origin_account_io_lease.as_mut() { - Some(lease) if lease.matches_binding(binding) && lease.provider_handle.is_some() => { - lease.active_io_count = lease.active_io_count.saturating_add(1); - HlsOriginAccountIoReserveOutcome::Joined - } - Some(lease) if lease.matches_binding(binding) && lease.acquiring => { - HlsOriginAccountIoReserveOutcome::Wait(Arc::clone(&lease.notify)) + match session.origin_account_io_lease.as_ref() { + Some(lease) if lease.matches_binding(binding) => { + if lease.acquiring.load(std::sync::atomic::Ordering::Acquire) { + HlsOriginAccountIoReserveOutcome::Wait(Arc::clone(&lease.notify)) + } else if lease.try_join() { + HlsOriginAccountIoReserveOutcome::Joined(Arc::clone(lease)) + } else { + let lease = Arc::new(HlsOriginAccountIoLease::acquiring(binding)); + session.origin_account_io_lease = Some(Arc::clone(&lease)); + HlsOriginAccountIoReserveOutcome::Acquire(lease) + } } Some(lease) if lease.is_active_or_acquiring() => HlsOriginAccountIoReserveOutcome::ReservedForOther, - Some(_) => { - session.origin_account_io_lease = None; - HlsOriginAccountIoReserveOutcome::Acquire - } - None => { - session.origin_account_io_lease = Some(HlsOriginAccountIoLease::acquiring(binding)); - HlsOriginAccountIoReserveOutcome::Acquire + _ => { + let lease = Arc::new(HlsOriginAccountIoLease::acquiring(binding)); + session.origin_account_io_lease = Some(Arc::clone(&lease)); + HlsOriginAccountIoReserveOutcome::Acquire(lease) } } } @@ -729,97 +916,150 @@ async fn store_acquired_hls_origin_account_io_handle( session: &HlsSessionHandle, binding: &HlsOriginAccountBinding, handle: ProviderHandle, + cleanup_permit: tokio::sync::mpsc::OwnedPermit, + mut pending_guard: PendingAcquireGuard, + deadline: Option, ) -> Result { - origin_io - .ctx - .active_provider - .refresh_provider_reservation(&binding.account_name, &binding.session_owner, origin_io.reservation_ttl_secs) - .await; - - let mut release_handle = None; - let mut notify_waiters = None; - { - let mut session = session.write().await; - if let Some(lease) = session - .origin_account_io_lease - .as_mut() - .filter(|lease| lease.matches_binding(binding) && lease.acquiring && lease.provider_handle.is_none()) - { - lease.provider_handle = Some(handle); - lease.acquiring = false; - lease.active_io_count = 1; - notify_waiters = Some(Arc::clone(&lease.notify)); - } else { - release_handle = Some(handle); + let playback_request_id = handle.playback_request_id; + // The handle carries the exact provider binding tag it was acquired under, so a + // later detach/clear can prove it still targets this binding, not a successor. + let provider_binding_tag = handle.binding_tag; + let mut handle_guard = ManagedHandleGuard::new(handle, cleanup_permit); + if let Some(request_id) = playback_request_id { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + clear_pending_hls_origin_account_io_lease(session, &pending_guard.lease, deadline).await; + pending_guard.disarm(); + return Err(HlsBoundAccountAcquireErrorKind::AcquireTimedOut); } + let lease_ref = PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }; + origin_io.ctx.active_provider.refresh_playback_lease( + &binding.account_name, + &lease_ref, + origin_io.reservation_ttl_secs, + ); } - if let Some(notify) = notify_waiters { - notify.notify_waiters(); + + let lease = Arc::clone(&pending_guard.lease); + + let store = async { + let mut session_guard = session.write().await; + let session_lease_matches = + session_guard.origin_account_io_lease.as_ref().is_some_and(|current| Arc::ptr_eq(current, &lease)); + let binding_still_current = session_guard.origin_account_binding.as_ref().is_some_and(|current| { + current.generation == binding.generation + && current.account_name == binding.account_name + && current.session_owner == binding.session_owner + && current.is_active() + }); + if !session_lease_matches + || !binding_still_current + || session_guard.is_gc_marked_for_removal() + || !matches!(hls_origin_account_status(&origin_io.ctx, binding), HlsOriginAccountStatus::Known) + { + return None; + } + + let (handle, permit) = handle_guard.take_resource()?; + match lease.provider_handle.lock() { + Ok(mut guard) => *guard = Some(handle), + Err(poisoned) => *poisoned.into_inner() = Some(handle), + } + match lease.cleanup_permit.lock() { + Ok(mut guard) => *guard = Some(permit), + Err(poisoned) => *poisoned.into_inner() = Some(permit), + } + lease.active_io_count.store(1, std::sync::atomic::Ordering::Release); + lease.acquiring.store(false, std::sync::atomic::Ordering::Release); + lease.notify.notify_waiters(); + if let Some(current) = session_guard.origin_account_binding.as_mut() { + current.playback_request_id = playback_request_id; + current.provider_binding_tag = provider_binding_tag; + } + let mut guard_binding = binding.clone(); + guard_binding.playback_request_id = playback_request_id; + guard_binding.provider_binding_tag = provider_binding_tag; + Some(HlsOriginAccountIoLeaseGuard::new(guard_binding, Arc::clone(&lease))) + }; + + let lease_guard = match deadline { + Some(deadline) => { + if let Ok(guard) = timeout(deadline.saturating_duration_since(Instant::now()), store).await { + guard + } else { + clear_pending_hls_origin_account_io_lease(session, &pending_guard.lease, Some(deadline)).await; + pending_guard.disarm(); + return Err(HlsBoundAccountAcquireErrorKind::AcquireTimedOut); + } + } + None => store.await, + }; + + if let Some(lease_guard) = lease_guard { + pending_guard.disarm(); + Ok(lease_guard) + } else { + clear_pending_hls_origin_account_io_lease(session, &pending_guard.lease, deadline).await; + pending_guard.disarm(); + Err(HlsBoundAccountAcquireErrorKind::StoreRace) } - if let Some(handle) = release_handle { - origin_io.ctx.connection_manager.release_provider_handle(Some(handle)).await; - origin_io.ctx.active_provider.clear_provider_reservation(&binding.session_owner).await; - return Err(HlsBoundAccountAcquireErrorKind::StoreRace); - } - Ok(HlsOriginAccountIoLeaseGuard { binding: binding.clone() }) } -async fn clear_pending_hls_origin_account_io_lease(session: &HlsSessionHandle, binding: &HlsOriginAccountBinding) { - let notify_waiters = { +async fn clear_pending_hls_origin_account_io_lease( + session: &HlsSessionHandle, + pending_lease: &Arc, + deadline: Option, +) { + pending_lease.acquiring.store(false, std::sync::atomic::Ordering::Release); + pending_lease.notify.notify_waiters(); + let clear = async { let mut session = session.write().await; - let notify = session - .origin_account_io_lease - .as_ref() - .filter(|lease| lease.matches_binding(binding) && lease.acquiring) - .map(|lease| Arc::clone(&lease.notify)); - if notify.is_some() { + if session.origin_account_io_lease.as_ref().is_some_and(|lease| Arc::ptr_eq(lease, pending_lease)) { session.origin_account_io_lease = None; } - notify }; - if let Some(notify) = notify_waiters { - notify.notify_waiters(); + match deadline { + Some(deadline) => { + let _ = timeout(deadline.saturating_duration_since(Instant::now()), clear).await; + } + None => clear.await, } } -pub async fn finish_hls_origin_io( - ctx: &HlsCtx, - binding: &HlsOriginAccountBinding, - provider_handle: Option, - reservation_ttl_secs: u64, -) { - ctx.connection_manager.release_provider_handle(provider_handle).await; - ctx.active_provider - .refresh_provider_reservation(&binding.account_name, &binding.session_owner, reservation_ttl_secs) - .await; -} - pub async fn finish_hls_origin_account_io( origin_io: &HlsOriginIoContext, session: &HlsSessionHandle, - guard: HlsOriginAccountIoLeaseGuard, + mut guard: HlsOriginAccountIoLeaseGuard, refresh_reservation: bool, ) { - let binding = guard.binding; - let mut provider_handle_to_release = None; + let binding = guard.binding().clone(); + let mut provider_resource_to_release = None; let mut should_refresh_reservation = false; - let mut should_clear_reservation = false; + let confirmed_ttl_secs; { let mut session = session.write().await; - if let Some(lease) = session - .origin_account_io_lease - .as_mut() - .filter(|lease| lease.matches_binding(&binding) && lease.active_io_count > 0) - { - lease.active_io_count = lease.active_io_count.saturating_sub(1); - if lease.active_io_count == 0 { - provider_handle_to_release = lease.provider_handle.take(); + + let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); + confirmed_ttl_secs = session.activity.last_delivered_media_at_ms.map_or(0, |last| { + last.saturating_add(origin_io.reservation_ttl_secs.saturating_mul(1000)) + .saturating_sub(now_ms) + .div_ceil(1000) + }); + + // Decrement directly on guard.lease, not session.origin_account_io_lease. + // The session may have been rebound or replaced between I/O start and finish. + // Only clear the session reference if it still points to this exact lease instance. + guard.disarm(); + if guard.lease.active_io_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel) == 1 { + provider_resource_to_release = guard.lease.take_resource(); + if session + .origin_account_io_lease + .as_ref() + .is_some_and(|session_lease| std::sync::Arc::ptr_eq(session_lease, &guard.lease)) + { session.origin_account_io_lease = None; - if refresh_reservation { - should_refresh_reservation = true; - } else { - should_clear_reservation = true; - } + } + if refresh_reservation && confirmed_ttl_secs > 0 { + should_refresh_reservation = true; } } @@ -827,6 +1067,7 @@ pub async fn finish_hls_origin_account_io( current.is_active() && current.account_name == binding.account_name && current.session_owner == binding.session_owner + && current.generation == binding.generation }) { let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); current.last_origin_io_at_ms = Some(now_ms); @@ -836,15 +1077,20 @@ pub async fn finish_hls_origin_account_io( } } - if let Some(provider_handle) = provider_handle_to_release { + if let Some((provider_handle, cleanup_permit)) = provider_resource_to_release { + let mut handle_guard = ManagedHandleGuard::new(provider_handle, cleanup_permit); if should_refresh_reservation { - finish_hls_origin_io(&origin_io.ctx, &binding, Some(provider_handle), origin_io.reservation_ttl_secs).await; - } else { - origin_io.ctx.connection_manager.release_provider_handle(Some(provider_handle)).await; - if should_clear_reservation { - origin_io.ctx.active_provider.clear_provider_reservation(&binding.session_owner).await; + if let Some(request_id) = binding.playback_request_id { + origin_io.ctx.active_provider.refresh_playback_lease( + &binding.account_name, + &PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }, + confirmed_ttl_secs, + ); } } + if let Some((handle, permit)) = handle_guard.take_resource() { + permit.send(CleanupEvent::ReleaseProviderHandle { handle: Some(handle) }); + } } } @@ -874,7 +1120,7 @@ mod tests { let mut session = HlsSession::new(HlsSessionKey::new(1, "lock-release"), b"secret", 1_000); let binding = HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); - session.origin_account_io_lease = Some(HlsOriginAccountIoLease::acquiring(&binding)); + session.origin_account_io_lease = Some(std::sync::Arc::new(HlsOriginAccountIoLease::acquiring(&binding))); let session = Arc::new(tokio::sync::RwLock::new(session)); let waiting_session = Arc::clone(&session); let waiting_binding = binding.clone(); @@ -889,7 +1135,7 @@ mod tests { let completion = tokio::spawn(async move { let notify = { let mut session = completing_session.write().await; - session.origin_account_io_lease.take().map(|lease| lease.notify) + session.origin_account_io_lease.take().map(|lease| std::sync::Arc::clone(&lease.notify)) }; if let Some(notify) = notify { notify.notify_waiters(); @@ -898,7 +1144,7 @@ mod tests { tokio::task::yield_now().await; assert!(completion.is_finished(), "the waiter must not retain the session lock"); completion.await.expect("controlled completion task"); - assert!(matches!(waiter.await.expect("controlled waiter task"), Ok(HlsOriginAccountIoSlot::Acquire))); + assert!(matches!(waiter.await.expect("controlled waiter task"), Ok(HlsOriginAccountIoSlot::Acquire(_)))); } #[test] @@ -909,6 +1155,54 @@ mod tests { assert!(!owner.contains('|')); } + #[test] + fn released_origin_io_lease_cannot_be_rejoined() { + let session = HlsSession::new(HlsSessionKey::new(1, "released-io"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + let lease = HlsOriginAccountIoLease::active_for_test(&binding, 0); + + assert!(!lease.try_join()); + assert_eq!(lease.active_io_count.load(std::sync::atomic::Ordering::Acquire), 0); + } + + #[test] + fn active_origin_io_lease_cannot_be_replaced_by_another_generation() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "generation-conflict"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + let conflicting = HlsOriginAccountBinding::rebound( + Arc::clone(&binding.input_name), + Arc::clone(&binding.account_name), + binding.session_owner.clone(), + binding.generation + 1, + 1_001, + ); + let lease = Arc::new(HlsOriginAccountIoLease::active_for_test(&binding, 1)); + session.origin_account_io_lease = Some(Arc::clone(&lease)); + + assert!(matches!( + super::try_reserve_hls_origin_account_io_slot(&mut session, &conflicting), + super::HlsOriginAccountIoReserveOutcome::ReservedForOther + )); + assert!(session.origin_account_io_lease.as_ref().is_some_and(|current| Arc::ptr_eq(current, &lease))); + assert_eq!(lease.active_io_count.load(std::sync::atomic::Ordering::Acquire), 1); + } + + #[test] + fn gc_marked_session_rejects_origin_io_reservation() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "gc-marked"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.mark_for_gc_removal(); + + assert!(matches!( + super::try_reserve_hls_origin_account_io_slot(&mut session, &binding), + super::HlsOriginAccountIoReserveOutcome::Unavailable + )); + assert!(session.origin_account_io_lease.is_none()); + } + #[test] fn origin_source_session_key_uses_input_and_stream_ref() { let direct = HlsOriginSource::new(7, Arc::from("input"), "80510", HlsOriginSourceKind::XtreamLive); @@ -945,18 +1239,18 @@ mod tests { let timing = HlsAccountOverlapTiming::from_target_duration_secs(Some(10)); assert_eq!( - classify_account_binding_protection(Some(1_000), 5_000, timing), + classify_account_binding_protection(Some(1_000), 5_000, &timing), HlsAccountBindingProtection::HardActive { until_ms: 11_000 } ); assert_eq!( - classify_account_binding_protection(Some(1_000), 20_000, timing), + classify_account_binding_protection(Some(1_000), 20_000, &timing), HlsAccountBindingProtection::SoftActive { reclaim_until_ms: 31_000 } ); assert_eq!( - classify_account_binding_protection(Some(1_000), 32_000, timing), + classify_account_binding_protection(Some(1_000), 32_000, &timing), HlsAccountBindingProtection::Expired ); - assert_eq!(classify_account_binding_protection(None, 1_000, timing), HlsAccountBindingProtection::NoMediaYet); + assert_eq!(classify_account_binding_protection(None, 1_000, &timing), HlsAccountBindingProtection::NoMediaYet); } #[test] @@ -1117,4 +1411,213 @@ mod tests { assert_eq!(state.consecutive_rebind_failures, 0); assert_eq!(state.next_rebind_allowed_at_ms, None); } + + #[tokio::test] + async fn origin_account_io_lease_guard_drop_decrements_count_and_cleans_up() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "guard-drop"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_io_lease = Some(std::sync::Arc::new(HlsOriginAccountIoLease { + account_name: Arc::clone(&binding.account_name), + session_owner: binding.session_owner.clone(), + provider_handle: std::sync::Mutex::new(None), + cleanup_permit: std::sync::Mutex::new(None), + acquiring: std::sync::atomic::AtomicBool::new(false), + active_io_count: std::sync::atomic::AtomicUsize::new(2), + notify: Arc::new(tokio::sync::Notify::new()), + generation: 0, + })); + let session = Arc::new(tokio::sync::RwLock::new(session)); + let guard = super::HlsOriginAccountIoLeaseGuard::new( + binding.clone(), + Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), + ); + drop(guard); + + let active_count = session + .read() + .await + .origin_account_io_lease + .as_ref() + .map(|l| l.active_io_count.load(std::sync::atomic::Ordering::Acquire)); + assert_eq!(active_count, Some(1), "drop must decrement active_io_count"); + } + + #[tokio::test] + async fn pending_acquire_guard_drop_clears_lease_and_notifies() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "pending-drop"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_io_lease = Some(std::sync::Arc::new(HlsOriginAccountIoLease::acquiring(&binding))); + let session = Arc::new(tokio::sync::RwLock::new(session)); + let guard = super::PendingAcquireGuard::new(Arc::clone( + session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap(), + )); + drop(guard); + + assert!( + !session + .read() + .await + .origin_account_io_lease + .as_ref() + .unwrap() + .acquiring + .load(std::sync::atomic::Ordering::Acquire), + "pending acquire drop must clear acquiring flag" + ); + } + + #[tokio::test] + async fn origin_account_io_lease_guard_drop_under_lock_contention_decrements_and_cleans_up() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "guard-contention"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_io_lease = Some(std::sync::Arc::new(HlsOriginAccountIoLease { + account_name: Arc::clone(&binding.account_name), + session_owner: binding.session_owner.clone(), + provider_handle: std::sync::Mutex::new(None), + cleanup_permit: std::sync::Mutex::new(None), + acquiring: std::sync::atomic::AtomicBool::new(false), + active_io_count: std::sync::atomic::AtomicUsize::new(2), + notify: Arc::new(tokio::sync::Notify::new()), + generation: 0, + })); + let session = Arc::new(tokio::sync::RwLock::new(session)); + let guard = super::HlsOriginAccountIoLeaseGuard::new( + binding.clone(), + Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), + ); + + let lock = session.read().await; + drop(guard); + assert_eq!( + lock.origin_account_io_lease.as_ref().map(|l| l.active_io_count.load(std::sync::atomic::Ordering::Acquire)), + Some(1), + "drop MUST synchronously modify active_io_count even when session is locked" + ); + drop(lock); + + for _ in 0..100 { + if session + .read() + .await + .origin_account_io_lease + .as_ref() + .map(|l| l.active_io_count.load(std::sync::atomic::Ordering::Acquire)) + == Some(1) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + let active_count = session + .read() + .await + .origin_account_io_lease + .as_ref() + .map(|l| l.active_io_count.load(std::sync::atomic::Ordering::Acquire)); + assert_eq!(active_count, Some(1), "deferred cleanup must decrement active_io_count"); + } + + #[tokio::test] + async fn pending_acquire_guard_drop_under_lock_contention_clears_lease_and_notifies() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "pending-contention"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + let notify = Arc::new(tokio::sync::Notify::new()); + session.origin_account_io_lease = Some(std::sync::Arc::new(HlsOriginAccountIoLease { + account_name: Arc::clone(&binding.account_name), + session_owner: binding.session_owner.clone(), + provider_handle: std::sync::Mutex::new(None), + cleanup_permit: std::sync::Mutex::new(None), + acquiring: std::sync::atomic::AtomicBool::new(true), + active_io_count: std::sync::atomic::AtomicUsize::new(0), + notify: Arc::clone(¬ify), + generation: 0, + })); + let session = Arc::new(tokio::sync::RwLock::new(session)); + let guard = super::PendingAcquireGuard::new(Arc::clone( + session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap(), + )); + + let lock = session.read().await; + let notified_future = notify.notified(); + drop(guard); + assert!( + !lock.origin_account_io_lease.as_ref().unwrap().acquiring.load(std::sync::atomic::Ordering::Acquire), + "drop MUST synchronously clear acquiring flag" + ); + drop(lock); + + let notified = tokio::time::timeout(std::time::Duration::from_secs(1), notified_future).await; + assert!(notified.is_ok(), "deferred cleanup must notify waiters"); + assert!( + !session + .read() + .await + .origin_account_io_lease + .as_ref() + .unwrap() + .acquiring + .load(std::sync::atomic::Ordering::Acquire), + "synchronous drop must clear acquiring flag" + ); + } + + /// The begin path must own the pending lease before the first cleanup-permit wait: + /// when the cleanup channel is saturated and the permit wait times out, the pending + /// lease is cleared instead of staying `acquiring` forever. + #[tokio::test] + async fn begin_cleanup_permit_timeout_allows_next_acquire() { + let ctx = crate::hls_ctx::HlsCtx::for_test(tuliprox_core::model::Config::default()); + let origin_io = super::HlsOriginIoContext { + ctx: ctx.clone(), + client_addr: "127.0.0.1:8080".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: tuliprox_session::ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: None, + }; + + let mut session = HlsSession::new(HlsSessionKey::new(1, "permit-timeout"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_binding = Some(binding.clone()); + let session = Arc::new(tokio::sync::RwLock::new(session)); + + // Saturate the control cleanup lane so `reserve_cleanup_permit` blocks until + // the bounded deadline fires. + let mut held_permits = Vec::new(); + while let Ok(permit) = tokio::time::timeout( + std::time::Duration::from_millis(1), + ctx.connection_manager.control_cleanup_tx().reserve_owned(), + ) + .await + { + held_permits.push(permit); + } + + let result = super::begin_hls_origin_account_io_bounded( + &origin_io, + &session, + &binding, + std::time::Duration::from_millis(50), + ) + .await; + assert!( + matches!(result, Err(HlsBoundAccountAcquireErrorKind::AcquireTimedOut)), + "a saturated cleanup queue must time out the permit wait, got {result:?}" + ); + + let pending_cleared = session + .read() + .await + .origin_account_io_lease + .as_ref() + .is_none_or(|lease| !lease.acquiring.load(std::sync::atomic::Ordering::Acquire)); + assert!(pending_cleared, "the pending lease must not stay acquiring after the permit timeout"); + } } diff --git a/backend/hls/src/playback.rs b/backend/hls/src/playback.rs index 0da3fe96a..27dfabd79 100644 --- a/backend/hls/src/playback.rs +++ b/backend/hls/src/playback.rs @@ -101,8 +101,8 @@ async fn validate_hls_access_lease_admission( false, ) .await; - if admission.permission == UserConnectionPermission::Exhausted - || (admission.permission == UserConnectionPermission::GracePeriod && admission.kind.is_none()) + if admission.permission() == UserConnectionPermission::Exhausted + || (admission.permission() == UserConnectionPermission::GracePeriod && admission.kind().is_none()) { let runtime_tail = begin_runtime_policy_denial( ctx, @@ -133,7 +133,7 @@ async fn validate_hls_access_lease_admission( &lease.user_session_token, ) .await - .or(admission.kind) + .or(admission.kind()) else { let runtime_tail = begin_runtime_policy_denial( ctx, diff --git a/backend/hls/src/refresh/mod.rs b/backend/hls/src/refresh/mod.rs index 1f115955e..6d54d0c3f 100644 --- a/backend/hls/src/refresh/mod.rs +++ b/backend/hls/src/refresh/mod.rs @@ -15,13 +15,13 @@ use super::{ availability_reevaluation::HlsRecoveryPressureGuardAccess, - begin_hls_origin_account_io, + begin_hls_origin_account_io_bounded, critical_handoff::{critical_handoff_terminal_response_is_current, retry_critical_handoff_state_access}, deterministic_conflict::HlsDeterministicTimelineConflict, finish_hls_origin_account_io, hls_ctx::WeakHlsCtx, - hls_manifest_recovery_log_fields, hls_origin_headers_with_provider_session, hls_origin_log_value, - hls_recovery_timing_policy, + hls_manifest_recovery_log_fields, hls_object_body_deadline, hls_origin_headers_with_provider_session, + hls_origin_log_value, hls_recovery_timing_policy, manifest_acceptance::{HlsManifestAcceptanceState, HlsManifestAcceptanceTrigger}, manifest_fetch::{ deterministic_conflict_receipt_is_current, deterministic_conflict_receipt_matches, @@ -415,7 +415,14 @@ async fn refresh_and_commit(mut request: OriginRefreshRequest, fetch_started_at_ let provider_lease = if let Some(origin_io) = request.origin_io.as_ref() { let binding = request.session.read().await.origin_account_binding.clone(); if let Some(binding) = binding { - match begin_hls_origin_account_io(origin_io, &request.session, &binding).await { + match begin_hls_origin_account_io_bounded( + origin_io, + &request.session, + &binding, + hls_object_body_deadline(request.origin_manifest_timeout_ms), + ) + .await + { Ok(guard) => { debug!( "HLS provider session lease joined for manifest refresh: provider={}", @@ -796,19 +803,19 @@ async fn release_preacquired_origin_provider_handle(request: &OriginRefreshReque let Some(origin_io) = request.origin_io.as_ref() else { return; }; - let Some(handle) = origin_io.take_preacquired_provider_handle().await else { + let Some(managed) = origin_io.take_preacquired_provider_handle() else { return; }; let binding = request.session.read().await.origin_account_binding.clone(); if let Some(binding) = binding { - origin_io.ctx.connection_manager.release_provider_handle(Some(handle)).await; debug!( "HLS provider handle released after manifest refresh: provider={} reason=refresh-not-started", sanitize_sensitive_info(binding.account_name.as_ref()) ); - } else { - origin_io.ctx.connection_manager.release_provider_handle(Some(handle)).await; } + // Drop the managed owner synchronously: the provider slot is released without a + // lossy cleanup message. + drop(managed); } async fn touch_refresh_origin_account_binding(request: &OriginRefreshRequest, reservation_refreshed: bool) { diff --git a/backend/hls/src/response.rs b/backend/hls/src/response.rs index c4176297c..a6c8af931 100644 --- a/backend/hls/src/response.rs +++ b/backend/hls/src/response.rs @@ -16,7 +16,7 @@ use axum::{ response::IntoResponse, }; use bytes::Bytes; -use futures::Stream; +use futures::{Stream, StreamExt}; use log::debug; use std::{ future::Future, @@ -225,6 +225,7 @@ pub fn finite_hls_media_head_response( #[derive(Clone)] struct CacheObject { + is_media: bool, key: K, access: Arc, content_type: String, @@ -324,9 +325,47 @@ pub struct HlsMediaActivityMarker { lease_identity: HlsMediaLeaseIdentity, completed_segment: Option, completion_scheduled: Option>, + active_provider: Option>, + session_owner: Option, + playback_request_id: Option, } impl HlsMediaActivityMarker { + /// Confirms only a polled, non-empty media body. Bytes retain their shared + /// backing storage; no payload copy or detached per-chunk task is needed. + pub fn confirm_media_response(&self, response: Response) -> Response { + let (parts, body) = response.into_parts(); + let mut marker = Some(self.clone()); + let stream = body.into_data_stream().then(move |chunk| { + let confirmation = if chunk.as_ref().is_ok_and(|bytes| !bytes.is_empty()) { marker.take() } else { None }; + async move { + if let Some(marker) = confirmation { + let outcome = marker + .manager + .mark_delivered_media_for_lease( + &marker.session, + &marker.lease_id, + &marker.proxy_session_id, + marker.lease_identity, + current_time_millis(), + ) + .await; + if matches!(outcome, HlsMediaActivityCommitOutcome::Committed) { + if let (Some(active_provider), Some(owner), Some(request_id)) = ( + marker.active_provider.as_ref(), + marker.session_owner.as_deref(), + marker.playback_request_id, + ) { + active_provider.confirm_identified_playback_activity(owner, request_id); + } + } + marker.log_uncommitted_activity(outcome, "media-delivery"); + } + chunk + } + }); + Response::from_parts(parts, Body::from_stream(stream)) + } pub fn new( manager: Arc, session: HlsSessionHandle, @@ -342,9 +381,24 @@ impl HlsMediaActivityMarker { lease_identity, completed_segment: None, completion_scheduled: None, + active_provider: None, + session_owner: None, + playback_request_id: None, } } + pub fn with_active_provider( + mut self, + active_provider: Arc, + session_owner: Option, + playback_request_id: Option, + ) -> Self { + self.active_provider = Some(active_provider); + self.session_owner = session_owner; + self.playback_request_id = playback_request_id; + self + } + async fn for_segment_request(mut self, proxy_seq: u64, requested_at_ms: u64) -> Option { // A terminal lease may still read the immutable READY live-tail segment // protected by its plan. That read marks terminal activity, but it must @@ -764,6 +818,11 @@ where insert_header_value(headers, header::CONTENT_RANGE, &format!("bytes {start}-{end}/{}", metadata.size)); } mark_response_as_uncompressed(&mut response); + if object.is_media { + if let Some(marker) = &context.media_activity_marker { + response = marker.confirm_media_response(response); + } + } Ok(response) } @@ -803,6 +862,7 @@ async fn lookup_segment_cache_object( } } CacheObjectLookup::Ready(CacheObject { + is_media: true, key: entry.cache_key.clone(), access: Arc::clone(&entry.access), content_type: entry.content_type.clone(), @@ -869,6 +929,7 @@ async fn lookup_map_cache_object( } } CacheObjectLookup::Ready(CacheObject { + is_media: false, key: entry.cache_key.clone(), access: Arc::clone(&entry.access), content_type: entry.content_type.clone(), @@ -929,6 +990,7 @@ async fn lookup_transient_object_cache_object( }; }; CacheObjectLookup::Ready(CacheObject { + is_media: matches!(resource_state.0, TransientResourceKind::Segment | TransientResourceKind::Part), key: entry.key, access: Arc::clone(&entry.access), content_type: entry.content_type, @@ -1637,6 +1699,7 @@ mod tests { segment_cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); let access = Arc::new(CacheAccessState::new()); let object = CacheObject { + is_media: true, key, access: Arc::clone(&access), content_type: "video/mp2t".to_string(), @@ -1669,6 +1732,7 @@ mod tests { let second = serve_cache_object( segment_cache, CacheObject { + is_media: true, log_context: CacheObjectLogContext { lease: "lease-b".to_string(), identity: test_log_identity(), @@ -1856,6 +1920,7 @@ mod tests { let response = serve_cache_object( segment_cache, CacheObject { + is_media: true, key, access, content_type: "video/mp2t".to_string(), diff --git a/backend/hls/src/session.rs b/backend/hls/src/session.rs index a6bad6654..08ac76ac8 100644 --- a/backend/hls/src/session.rs +++ b/backend/hls/src/session.rs @@ -88,8 +88,10 @@ pub enum HlsSegmentFailureTransition { pub struct HlsSessionActivity { pub last_authorized_manifest_at_ms: Option, pub last_authorized_media_at_ms: Option, + /// Client media responses, excluding manifests and origin prefetches. + pub last_delivered_media_at_ms: Option, pub active_access_lease_count: usize, - pub active_origin_work_count: usize, + pub active_origin_work_count: std::sync::Arc, pub origin_work_generation: u64, pub media_readiness_generation: u64, } @@ -249,7 +251,7 @@ pub struct HlsSession { pub proxy_session_id: ProxySessionId, pub origin_source: HlsOriginSource, pub origin_account_binding: Option, - pub origin_account_io_lease: Option, + pub origin_account_io_lease: Option>, pub origin_account_rebind: HlsOriginAccountRebindState, pub effective_origin_acquire_policy: Option, pub mode: HlsSessionMode, @@ -516,13 +518,13 @@ impl HlsSession { classify_account_binding_protection( self.activity.last_authorized_media_at_ms, now_ms, - self.account_overlap_timing(), + &self.account_overlap_timing(), ) } pub fn should_refresh_origin_reservation(&self, now_ms: u64) -> bool { !matches!(self.account_binding_protection(now_ms), HlsAccountBindingProtection::Expired) - || self.activity.active_origin_work_count > 0 + || self.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 } pub fn reconcile_effective_origin_acquire_policy( @@ -587,7 +589,7 @@ impl HlsSession { if self.idle_expiry_due_at_ms(session_idle_timeout_ms) > now_ms { return false; } - self.activity.active_origin_work_count == 0 + self.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) == 0 && self.active_segment_fetches == 0 && self.active_map_fetches == 0 && !self.origin_refresh.in_flight @@ -753,12 +755,12 @@ impl HlsSession { } pub fn start_origin_work(&mut self) -> u64 { - self.activity.active_origin_work_count = self.activity.active_origin_work_count.saturating_add(1); + self.activity.active_origin_work_count.fetch_add(1, std::sync::atomic::Ordering::AcqRel); self.activity.origin_work_generation } pub fn finish_origin_work(&mut self, started_generation: u64) -> bool { - self.activity.active_origin_work_count = self.activity.active_origin_work_count.saturating_sub(1); + self.activity.active_origin_work_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); started_generation == self.activity.origin_work_generation } @@ -1394,7 +1396,7 @@ mod tests { session.invalidate_queued_origin_work(); assert!(!session.finish_origin_work(started_generation)); - assert_eq!(session.activity.active_origin_work_count, 0); + assert_eq!(session.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), 0); assert_eq!(session.activity.origin_work_generation, 1); } diff --git a/backend/hls/src/transient_fetcher.rs b/backend/hls/src/transient_fetcher.rs index e2ccfaf30..0715d013f 100644 --- a/backend/hls/src/transient_fetcher.rs +++ b/backend/hls/src/transient_fetcher.rs @@ -1,5 +1,5 @@ use super::{ - build_hls_origin_resource_headers_with_client_range, finish_hls_origin_account_io, hls_client_body_send_deadline, + build_hls_origin_resource_headers_with_client_range, hls_client_body_send_deadline, refresh_hls_client_body_send_deadline, resource_fetch::{log_hls_resource_body_failure, HlsResourceFetchLogContext}, run_hls_origin_resource_retry_loop_with_attempt_prepare, @@ -328,6 +328,8 @@ pub struct HlsTransientCacheCommitContext { pub segment_cache: Arc, pub segment_repair: Arc, pub session: HlsSessionHandle, + pub proxy_session_id: ProxySessionId, + pub log_identity: super::HlsLogIdentity, pub access_lease_id: HlsAccessLeaseId, pub resource: TransientResourceRef, pub resource_file: TransientResourceFile, @@ -389,10 +391,8 @@ async fn commit_hls_transient_origin_response_attempt( .map(str::to_string) .or_else(|| context.resource.content_type_hint.clone()) .unwrap_or_else(|| "application/octet-stream".to_string()); - let (proxy_session_id, log_identity) = { - let session = context.session.read().await; - (session.proxy_session_id.clone(), super::HlsLogIdentity::from_session(&session)) - }; + let proxy_session_id = context.proxy_session_id.clone(); + let log_identity = context.log_identity.clone(); let repair_context = HlsSegmentRepairObjectContext { source: HlsSegmentRepairSource::Transient, log_identity, @@ -592,7 +592,7 @@ fn hls_transient_direct_body( Box::pin(sleep(hls_client_body_send_deadline())), false, ), - move |(mut stream, guard, origin_io_guard, mut finalizer, mut send_deadline, finished)| { + move |(mut stream, guard, mut origin_io_guard, mut finalizer, mut send_deadline, finished)| { let log_identity = log_identity.clone(); let resource_id = resource_id.clone(); async move { @@ -630,6 +630,9 @@ fn hls_transient_direct_body( } Ok(None) => { finalizer.finish(HlsTransientDirectStreamOutcome::CleanEof).await; + if let Some(guard) = origin_io_guard.take() { + guard.finish_clean().await; + } None } Err(_) => { @@ -772,58 +775,65 @@ impl Drop for HlsTransientReadGuard { pub struct HlsTransientOriginIoGuard { session: HlsSessionHandle, + active_origin_work_count: std::sync::Arc, origin_io: HlsOriginIoContext, lease_guard: Option, started_generation: u64, + origin_work_finished: bool, } impl HlsTransientOriginIoGuard { pub fn new( session: HlsSessionHandle, + active_origin_work_count: std::sync::Arc, origin_io: HlsOriginIoContext, lease_guard: HlsOriginAccountIoLeaseGuard, started_generation: u64, ) -> Self { - Self { session, origin_io, lease_guard: Some(lease_guard), started_generation } + Self { + session, + active_origin_work_count, + origin_io, + lease_guard: Some(lease_guard), + started_generation, + origin_work_finished: false, + } + } + + pub async fn finish_clean(mut self) { + let generation_valid = { + let mut session = self.session.write().await; + let valid = session.finish_origin_work(self.started_generation); + self.origin_work_finished = true; + valid + }; + let refresh_reservation = if generation_valid { + self.session + .read() + .await + .should_refresh_origin_reservation(chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default()) + } else { + false + }; + if let Some(lease_guard) = self.lease_guard.take() { + crate::origin::finish_hls_origin_account_io( + &self.origin_io, + &self.session, + lease_guard, + refresh_reservation, + ) + .await; + } } } impl Drop for HlsTransientOriginIoGuard { fn drop(&mut self) { - let Some(lease_guard) = self.lease_guard.take() else { + if self.origin_work_finished { return; - }; - let session = Arc::clone(&self.session); - let origin_io = self.origin_io.clone(); - let started_generation = self.started_generation; - // Decrement the origin work count synchronously when the lock is free so an - // immediate retry is not rejected by the admission check (active_origin_work_count > 0) - // while the spawned cleanup is still pending - let pre_finished = session.try_write().map(|mut guard| guard.finish_origin_work(started_generation)).ok(); - tokio::spawn(async move { - let generation_valid = if let Some(valid) = pre_finished { - valid - } else { - let mut session = session.write().await; - session.finish_origin_work(started_generation) - }; - let refresh_reservation = if generation_valid { - session.read().await.should_refresh_origin_reservation( - chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(), - ) - } else { - false - }; - finish_hls_origin_account_io(&origin_io, &session, lease_guard, refresh_reservation).await; - let mut session = session.write().await; - if let Some(binding) = session.origin_account_binding.as_mut() { - let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); - binding.last_origin_io_at_ms = Some(now_ms); - if refresh_reservation { - binding.last_reservation_refresh_at_ms = Some(now_ms); - } - } - }); + } + self.origin_work_finished = true; + self.active_origin_work_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); } } @@ -846,6 +856,7 @@ mod tests { HlsTransientOriginIoGuard, HlsTransientResourceLeaseContext, }; use crate::{ + origin::{HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginAccountIoLeaseGuard, HlsOriginIoContext}, HlsAccessLeaseId, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsPublishedTransientResourceIds, HlsSegmentCache, HlsSegmentFailureObject, HlsSegmentRepairManager, HlsSession, HlsSessionHandle, HlsSessionKey, HlsSessionStore, ProxySessionId, SegmentFetchPolicy, TransientObjectCacheKey, TransientObjectCacheStatus, @@ -1485,6 +1496,8 @@ mod tests { segment_cache: Arc::clone(&self.segment_cache), segment_repair: Arc::clone(&self.segment_repair), session: Arc::clone(&self.session), + proxy_session_id: ProxySessionId(String::from("test-proxy-session")), + log_identity: self.log_identity.clone(), access_lease_id: HlsAccessLeaseId("transient-content-coding-test".to_string()), resource: self.resource.clone(), resource_file: self.resource_file.clone(), @@ -2104,4 +2117,106 @@ mod tests { assert!(request.contains("accept-encoding: identity")); assert!(!request.contains("\r\nrange:")); } + + #[tokio::test] + async fn transient_origin_io_guard_finish_clean_decrements_work_once() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "clean-finish"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_io_lease = + Some(std::sync::Arc::new(HlsOriginAccountIoLease::active_for_test(&binding, 2))); + session.origin_account_binding = Some(binding.clone()); + session.activity.active_origin_work_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(2)); + let started_gen = session.activity.origin_work_generation; + let session = Arc::new(tokio::sync::RwLock::new(session)); + let ctx = crate::hls_ctx::HlsCtx::for_test(tuliprox_core::model::Config::default()); + + let lease_guard = HlsOriginAccountIoLeaseGuard::new( + binding.clone(), + Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), + ); + let origin_io = HlsOriginIoContext { + ctx: ctx.clone(), + client_addr: "127.0.0.1:8080".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: tuliprox_session::ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: Some(started_gen), + }; + + let guard = HlsTransientOriginIoGuard::new( + Arc::clone(&session), + Arc::clone(&session.try_read().unwrap().activity.active_origin_work_count), + origin_io, + lease_guard, + started_gen, + ); + + guard.finish_clean().await; + assert_eq!( + session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), + 1, + "finish_clean and subsequent drop must decrement active_origin_work_count exactly once" + ); + } + + #[tokio::test] + async fn transient_origin_io_guard_drop_under_lock_contention_decrements_work() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "contention-drop"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_io_lease = + Some(std::sync::Arc::new(HlsOriginAccountIoLease::active_for_test(&binding, 2))); + session.origin_account_binding = Some(binding.clone()); + session.activity.active_origin_work_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(2)); + let started_gen = session.activity.origin_work_generation; + let session = Arc::new(tokio::sync::RwLock::new(session)); + let ctx = crate::hls_ctx::HlsCtx::for_test(tuliprox_core::model::Config::default()); + + let lease_guard = HlsOriginAccountIoLeaseGuard::new( + binding.clone(), + Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), + ); + let origin_io = HlsOriginIoContext { + ctx: ctx.clone(), + client_addr: "127.0.0.1:8080".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: tuliprox_session::ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: Some(started_gen), + }; + + let guard = HlsTransientOriginIoGuard::new( + Arc::clone(&session), + Arc::clone(&session.try_read().unwrap().activity.active_origin_work_count), + origin_io, + lease_guard, + started_gen, + ); + + let lock = session.read().await; + drop(guard); + assert_eq!( + lock.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), + 1, + "drop MUST synchronously decrement active_origin_work_count" + ); + drop(lock); + + for _ in 0..100 { + if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + assert_eq!( + session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), + 1, + "deferred cleanup must decrement active_origin_work_count under contention" + ); + } } diff --git a/backend/iptv/src/lib.rs b/backend/iptv/src/lib.rs index d9d8ebddd..72f5cec34 100644 --- a/backend/iptv/src/lib.rs +++ b/backend/iptv/src/lib.rs @@ -1,3 +1,8 @@ +// The `PlaylistProvider` trait returns `impl Future + Send` from a generic impl; proving +// `Send` for that future recurses deeply through the Xtream fetch path and overflows the +// default recursion limit on newer nightlies. Raise it rather than flatten the generics. +#![recursion_limit = "256"] + pub mod capabilities; pub mod capability_store; pub mod clock; diff --git a/backend/library/Cargo.toml b/backend/library/Cargo.toml index dfe83fe98..41c4b9774 100644 --- a/backend/library/Cargo.toml +++ b/backend/library/Cargo.toml @@ -20,7 +20,6 @@ blake3.workspace = true bytes.workspace = true chrono.workspace = true fancy-regex = "0.17.0" -filetime = "0.2.29" log.workspace = true paste.workspace = true path-clean.workspace = true @@ -37,4 +36,5 @@ uuid = { version = "1.26.0", features = ["v4"] } [dev-dependencies] tempfile = "3.27.0" +filetime = "0.2.29" tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/backend/messaging/src/lib.rs b/backend/messaging/src/lib.rs index 2f37cee88..5a35b3179 100644 --- a/backend/messaging/src/lib.rs +++ b/backend/messaging/src/lib.rs @@ -8,6 +8,12 @@ //! kinds are an open set too - see [`shared::model::notification`]. Adding //! either no longer means editing a match in this file. +// The notification outbox spawns a generic async worker whose `Send` proof makes the +// trait solver recurse deeply; the default recursion limit overflows on newer +// nightlies (recursion_depth_exceeding_limit). Raise it rather than restructure the +// worker's generic bounds. +#![recursion_limit = "256"] + pub mod channel; pub mod channels; pub mod outbox; diff --git a/backend/metadata/src/manager.rs b/backend/metadata/src/manager.rs index 7c76dafa9..8251055da 100644 --- a/backend/metadata/src/manager.rs +++ b/backend/metadata/src/manager.rs @@ -2958,8 +2958,7 @@ impl InputWorker { // Reserve provider capacity only for actual probe work (ffprobe paths). let provider_handle = if needs_probe_connection { - let Some(handle) = ctx.active_provider.acquire_connection_for_probe(input_name, probe_priority).await - else { + let Some(handle) = ctx.active_provider.acquire_connection_for_probe(input_name, probe_priority) else { debug_if_enabled!("No provider connection available for background task {}, skipping...", task); return Err(shared::error::TuliproxError::Config(TASK_ERR_NO_CONNECTION.to_string())); }; @@ -3070,7 +3069,7 @@ impl InputWorker { }; if provider_handle.is_some() { - ctx.connection_manager.release_provider_handle(provider_handle).await; + ctx.connection_manager.release_provider_handle(provider_handle); } match res { Ok((tmdb_and_date_present, probe_pending)) => { diff --git a/backend/parser/src/m3u.rs b/backend/parser/src/m3u.rs index 18d58a257..b26203075 100644 --- a/backend/parser/src/m3u.rs +++ b/backend/parser/src/m3u.rs @@ -187,6 +187,8 @@ fn classify_token(t: &str) -> M3uToken { 4 => { if eq_ascii(bytes, b"cuid") { M3uToken::ProviderId + } else if eq_ascii(bytes, b"type") { + M3uToken::TvgType } else { M3uToken::Unknown } diff --git a/backend/processing/src/processor/probe_handle_guard.rs b/backend/processing/src/processor/probe_handle_guard.rs index 7e1ae4a67..7c3053692 100644 --- a/backend/processing/src/processor/probe_handle_guard.rs +++ b/backend/processing/src/processor/probe_handle_guard.rs @@ -1,38 +1,22 @@ use std::sync::Arc; use tuliprox_core::model::ProviderHandle; -use tuliprox_session::ActiveProviderManager; +use tuliprox_session::{ActiveProviderManager, ManagedProviderHandle}; -pub struct ProbeHandleGuard { - manager: Arc, - handle: Option, -} +/// Thin wrapper around `ManagedProviderHandle` for probe allocations. It preserves the +/// same synchronous release contract without duplicating the release/drop logic. +pub struct ProbeHandleGuard(ManagedProviderHandle); impl ProbeHandleGuard { pub fn new(manager: &Arc, handle: ProviderHandle) -> Self { - Self { manager: Arc::clone(manager), handle: Some(handle) } + Self(ManagedProviderHandle::new(Arc::clone(manager), handle)) } #[inline] - pub fn handle(&self) -> Option<&ProviderHandle> { self.handle.as_ref() } + pub fn handle(&self) -> Option<&ProviderHandle> { self.0.handle() } - pub async fn release(mut self) { - if let Some(handle) = self.handle.take() { - self.manager.release_handle(&handle).await; - } - } -} - -impl Drop for ProbeHandleGuard { - fn drop(&mut self) { - let Some(handle) = self.handle.take() else { - return; - }; - let manager = Arc::clone(&self.manager); - if let Ok(runtime_handle) = tokio::runtime::Handle::try_current() { - runtime_handle.spawn(async move { - manager.release_handle(&handle).await; - }); - } + pub fn release(self) { + // The inner managed handle releases the slot synchronously on drop. + drop(self); } } @@ -105,15 +89,37 @@ mod tests { let handle = manager .acquire_connection_for_probe(&input_name, default_probe_user_priority()) - .await .expect("probe allocation should succeed"); - assert_eq!(manager.get_provider_connections_count().await, 1); + assert_eq!(manager.get_provider_connections_count(), 1); let guard = ProbeHandleGuard::new(&manager, handle); drop(guard); - tokio::task::yield_now().await; - tokio::task::yield_now().await; - assert_eq!(manager.get_provider_connections_count().await, 0); + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[test] + fn probe_handle_guard_releases_provider_slot_outside_tokio_runtime() { + let rt = + tokio::runtime::Builder::new_current_thread().enable_all().build().expect("tokio runtime should build"); + + let (manager, guard) = rt.block_on(async { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name = "provider_1".intern(); + + let handle = manager + .acquire_connection_for_probe(&input_name, default_probe_user_priority()) + .expect("probe allocation should succeed"); + assert_eq!(manager.get_provider_connections_count(), 1); + + let guard = ProbeHandleGuard::new(&manager, handle); + (manager, guard) + }); + + // Drop outside of any tokio runtime context: + drop(guard); + assert_eq!(manager.get_provider_connections_count(), 0); } } diff --git a/backend/processing/src/processor/stalker_refresh.rs b/backend/processing/src/processor/stalker_refresh.rs index dd9167f03..5009cc3f3 100644 --- a/backend/processing/src/processor/stalker_refresh.rs +++ b/backend/processing/src/processor/stalker_refresh.rs @@ -1213,7 +1213,7 @@ mod tests { finish_completed_refresh_checked(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0)) .await?; - let StalkerRefreshOutcome::Failed { quality_acceptances, error: _, .. } = reports.into_outcome() else { + let StalkerRefreshOutcome::Failed { quality_acceptances, .. } = reports.into_outcome() else { return Err("expected failed Stalker completion with retained reports".into()); }; assert_eq!( @@ -1258,7 +1258,7 @@ mod tests { finish_completed_refresh_checked(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0)) .await?; - let StalkerRefreshOutcome::Failed { quality_acceptances, error: _, .. } = reports.into_outcome() else { + let StalkerRefreshOutcome::Failed { quality_acceptances, .. } = reports.into_outcome() else { return Err("expected failed Stalker completion with retained reports".into()); }; assert_eq!(quality_acceptances.len(), 1); diff --git a/backend/processing/src/processor/stream_probe.rs b/backend/processing/src/processor/stream_probe.rs index 047d9f457..ab401b97c 100644 --- a/backend/processing/src/processor/stream_probe.rs +++ b/backend/processing/src/processor/stream_probe.rs @@ -239,7 +239,6 @@ async fn prepare_generic_stream_metadata( } else { active_provider .acquire_connection_for_probe(&input.name, probe_priority) - .await .map(|handle| ProbeHandleGuard::new(active_provider, handle)) }; @@ -286,7 +285,7 @@ async fn prepare_generic_stream_metadata( }; if let Some(handle) = acquired_handle { - handle.release().await; + handle.release(); } let (raw_video, raw_audio, stats) = match probe_data { diff --git a/backend/processing/src/processor/xtream_series.rs b/backend/processing/src/processor/xtream_series.rs index e2f32d53d..f4553daaa 100644 --- a/backend/processing/src/processor/xtream_series.rs +++ b/backend/processing/src/processor/xtream_series.rs @@ -120,10 +120,7 @@ async fn playlist_resolve_series_info, AllocationId); +const PREEMPTION_COMPLETION_TIMEOUT: Duration = Duration::from_millis(1500); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ConnectionKind { Normal, @@ -52,22 +59,69 @@ fn is_better_preemption_candidate(current: Option, candidat #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum PriorityOwner { - Single(ClientConnectionId), + Single(AllocationId), Shared(SharedConnectionId), } +/// Playback identity of one provider acquisition. +/// +/// `owner` is a stable playback identity (usually the user session token), never a +/// socket address and never a per-attempt random suffix in a shared family. Retries +/// of one playback resolve to the same owner and therefore to one capacity slot, +/// while two players behind a reverse proxy keep independent owners. +#[derive(Debug, Clone, Copy)] +pub struct PlaybackLeaseRef<'a> { + pub owner: &'a str, + pub kind: PlaybackKind, + pub request_id: PlaybackRequestId, +} + +impl<'a> PlaybackLeaseRef<'a> { + pub fn new(owner: &'a str, kind: PlaybackKind) -> Self { + Self { owner, kind, request_id: PlaybackRequestId::next() } + } + + /// Compatibility view of a bare session owner: one request, live TS semantics. + pub fn for_session_owner(owner: &'a str) -> Self { Self::new(owner, PlaybackKind::LiveTs) } +} + struct AcquireProviderParams<'a> { addr: &'a SocketAddr, priority: i8, kind: ConnectionKind, - session_owner: Option<&'a str>, + lease: Option>, +} + +struct ProviderAllocationGuard(Option); + +impl ProviderAllocationGuard { + fn new(allocation: ProviderAllocation) -> Self { Self(Some(allocation)) } + + fn allocation(&self) -> &ProviderAllocation { self.0.as_ref().unwrap_or(&ProviderAllocation::Exhausted) } + + fn take(&mut self) -> ProviderAllocation { self.0.take().unwrap_or(ProviderAllocation::Exhausted) } +} + +impl Drop for ProviderAllocationGuard { + fn drop(&mut self) { + if let Some(allocation) = self.0.take() { + allocation.release(); + } + } +} + +impl AcquireProviderParams<'_> { + #[inline] + fn session_owner(&self) -> Option<&str> { self.lease.map(|lease| lease.owner) } } #[derive(Debug, Clone)] struct SharedAllocation { allocation_id: AllocationId, allocation: ProviderAllocation, - connections: HashMap, + /// Keyed by unique subscriber id, never by socket: two external clients behind one + /// reverse proxy must not collapse into a single entry. + connections: HashMap, priority: i8, kind: ConnectionKind, created_at: Instant, @@ -77,55 +131,316 @@ struct SharedAllocation { #[derive(Debug, Clone)] struct ActiveConnectionInfo { + allocation_id: AllocationId, + client_addr: SocketAddr, allocation: ProviderAllocation, // Used to signal preemption to the consumer of this connection cancel_token: CancellationToken, + completion_token: CancellationToken, + close_reason: Arc, + lifecycle: ConnectionLifecycle, + has_body_owner: bool, + reaper_spawned: bool, + open_generation: u64, created_at: Instant, priority: i8, kind: ConnectionKind, session_owner: Option>, + playback_request_id: Option, } #[derive(Debug, Clone, Copy)] struct SharedSubscriber { priority: i8, kind: ConnectionKind, + /// Transport metadata and socket-wide close target only; never an identity key. + addr: SocketAddr, } -#[derive(Debug, Clone)] -struct ProviderReservation { - expires_at: TokioInstant, -} - -type ProviderReservations = HashMap, HashMap, ProviderReservation>>; - #[derive(Debug, Clone, Default)] struct SharedConnections { by_key: HashMap, SharedAllocation>, - key_by_addr: HashMap>, + key_by_subscriber: HashMap>, shared_by_allocation_id: HashMap>, } #[derive(Debug, Clone, Default)] struct Connections { - // Map Addr -> AllocationID -> Allocation Info - single: HashMap>, + // Primary map keyed by AllocationId, never by socket address. + single: HashMap, + // Secondary index for socket-wide transport close/kick actions. + single_by_addr: HashMap>, shared: SharedConnections, // Index to quickly find connections by provider name for preemption - // ProviderName -> Set<(ClientConnectionId, AllocationId)> - by_provider: HashMap, HashSet<(ClientConnectionId, AllocationId)>>, + // ProviderName -> Set + by_provider: HashMap, HashSet>, + // Index to find every allocation (single and shared) owned by a playback session. + // SessionOwner -> Set + by_owner: HashMap, HashSet>, // Priority index per provider alias for O(log n) victim lookup // ProviderName -> BTreeMap priority_index: HashMap, BTreeMap>, soft_priority_index: HashMap, BTreeMap>, } -pub struct ActiveProviderManager { +pub struct ManagedProviderHandle { + manager: Arc, + handle: Option, +} + +impl std::fmt::Debug for ManagedProviderHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ManagedProviderHandle").field("handle", &self.handle).finish_non_exhaustive() + } +} + +impl ManagedProviderHandle { + #[inline] + pub fn manager(&self) -> &Arc { &self.manager } + + #[inline] + pub fn new(manager: Arc, handle: ProviderHandle) -> Self { + Self { manager, handle: Some(handle) } + } + + #[inline] + pub fn handle(&self) -> Option<&ProviderHandle> { self.handle.as_ref() } + + #[inline] + pub fn take(&mut self) -> Option { self.handle.take() } + + #[inline] + pub fn disarm(&mut self) -> Option { self.handle.take() } + + pub fn mark_opening(&self) { + if let Some(handle) = self.handle.as_ref() { + self.manager.mark_opening(handle.allocation_id); + } + } + + pub fn register_body_owner(&self) -> bool { + if let Some(handle) = self.handle.as_ref() { + self.manager.register_body_owner(handle.allocation_id) + } else { + false + } + } + + pub fn renew_opening_tokens(&mut self) -> Option<(CancellationToken, CancellationToken)> { + if let Some(ref mut handle) = self.handle { + if let Some((cancel, completion, gen)) = self.manager.renew_opening_tokens(handle.allocation_id) { + handle.cancel_token = Some(cancel.clone()); + handle.completion_token = Some(completion.clone()); + handle.open_generation = gen; + return Some((cancel, completion)); + } + } + None + } +} + +impl Drop for ManagedProviderHandle { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + self.manager.release_handle(&handle); + } + } +} + +pub struct ActiveProviderManagerCore { + // Serializes capacity transitions across counters, allocation indices and leases. + capacity_transition: std::sync::Mutex<()>, + is_shutting_down: AtomicBool, + shutdown_token: CancellationToken, providers: ProviderLineupManager, - connections: RwLock, - reservations: RwLock, + connections: std::sync::RwLock, + leases: std::sync::RwLock, next_allocation_id: AtomicU64, - shared_stream_manager: OnceLock>, +} + +impl ActiveProviderManagerCore { + /// Serialises capacity transitions (acquire/release/reclassify) so a reclassify + /// cannot race an acquire or release. + /// + /// Lock-order invariant: this lock must be acquired **before** the connections + /// lock (`read_connections`/`write_connections`). No path may hold the + /// connections lock and then call this; doing so can deadlock. + fn lock_capacity_transition(&self) -> std::sync::MutexGuard<'_, ()> { + match self.capacity_transition.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned active-provider capacity transition lock"); + poisoned.into_inner() + } + } + } + + fn read_connections(&self) -> RwLockReadGuard<'_, Connections> { + match self.connections.read() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned active-provider connection lock"); + poisoned.into_inner() + } + } + } + + fn write_connections(&self) -> RwLockWriteGuard<'_, Connections> { + match self.connections.write() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned active-provider connection lock"); + poisoned.into_inner() + } + } + } + + fn write_leases(&self) -> RwLockWriteGuard<'_, ProviderLeaseTable> { + match self.leases.write() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned provider-lease lock"); + poisoned.into_inner() + } + } + } + + #[cfg(test)] + fn read_leases(&self) -> RwLockReadGuard<'_, ProviderLeaseTable> { + match self.leases.read() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } + + pub fn complete_release(&self, alloc_id: AllocationId) { self.complete_release_with_generation(alloc_id, None); } + + pub fn complete_release_with_generation(&self, alloc_id: AllocationId, generation: Option) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + Self::complete_release_locked(&mut connections, alloc_id, generation); + } + + fn complete_release_locked( + connections: &mut Connections, + alloc_id: AllocationId, + expected_generation: Option, + ) -> Option { + if let Some(info) = connections.single.get(&alloc_id) { + if let Some(expected_gen) = expected_generation { + if info.open_generation != expected_gen { + return None; + } + } + } + if let Some(mut info) = connections.single.remove(&alloc_id) { + ActiveProviderManager::unindex_owner(connections, alloc_id, info.session_owner.as_ref()); + if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { + set.remove(&alloc_id); + if set.is_empty() { + connections.single_by_addr.remove(&info.client_addr); + } + } + if !info.allocation.is_unlimited_provider() { + if let Some(name) = info.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&alloc_id); + } + ActiveProviderManager::remove_priority_entry( + connections, + &name, + &(info.priority, Reverse(info.created_at), alloc_id), + info.kind, + ); + } + } + info.lifecycle = ConnectionLifecycle::Closed; + info.cancel_token.cancel(); + info.allocation.release(); + Some(info.allocation) + } else { + None + } + } + + fn plan_single_release_locked(connections: &mut Connections, alloc_id: AllocationId) -> ReleaseAction { + let Some(info) = connections.single.get_mut(&alloc_id) else { + return ReleaseAction::None; + }; + if info.lifecycle == ConnectionLifecycle::Closed { + return ReleaseAction::None; + } + + info.cancel_token.cancel(); + + let completed = info.completion_token.is_cancelled(); + let needs_completion_wait = (info.lifecycle == ConnectionLifecycle::Closing + || info.lifecycle == ConnectionLifecycle::Opening + || info.has_body_owner) + && !completed; + + if needs_completion_wait { + info.lifecycle = ConnectionLifecycle::Closing; + let client_addr = info.client_addr; + let prio = info + .allocation + .get_provider_name() + .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)); + let already_spawned = info.reaper_spawned; + info.reaper_spawned = true; + let gen = info.open_generation; + let completion_token = info.completion_token.clone(); + + if let Some((name, key, kind)) = prio { + ActiveProviderManager::remove_priority_entry(connections, &name, &key, kind); + } + if let Some(set) = connections.single_by_addr.get_mut(&client_addr) { + set.remove(&alloc_id); + if set.is_empty() { + connections.single_by_addr.remove(&client_addr); + } + } + if already_spawned { + ReleaseAction::None + } else { + ReleaseAction::Wait(alloc_id, gen, completion_token) + } + } else { + let gen = info.open_generation; + Self::complete_release_locked(connections, alloc_id, Some(gen)); + ReleaseAction::Immediate + } + } +} + +#[derive(Debug)] +enum ReleaseAction { + None, + Immediate, + Wait(AllocationId, u64, CancellationToken), +} + +#[derive(Debug)] +pub(crate) enum PreemptionOutcome { + Acquired(ProviderAllocation), + /// Victim body is still draining. The optional pair is `(alloc_id, open_generation)` of the + /// victim for an idempotent self-release attempt once the token fires; `None` for shared-stream + /// teardowns where release happens inside the spawned task and no `complete_release_locked` call + /// is needed. + PendingCompletion(Option<(AllocationId, u64)>, CancellationToken), + Exhausted, +} + +pub struct ActiveProviderManager { + core: Arc, + shared_stream_manager: OnceLock>, +} + +impl std::ops::Deref for ActiveProviderManager { + type Target = ActiveProviderManagerCore; + + #[inline] + fn deref(&self) -> &Self::Target { &self.core } } impl ActiveProviderManager { @@ -158,14 +473,32 @@ impl ActiveProviderManager { } } + fn index_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc>) { + if let Some(owner) = owner { + connections.by_owner.entry(Arc::clone(owner)).or_default().insert(allocation_id); + } + } + + fn unindex_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc>) { + if let Some(owner) = owner { + let remove = connections.by_owner.get_mut(owner).is_some_and(|set| { + set.remove(&allocation_id); + set.is_empty() + }); + if remove { + connections.by_owner.remove(owner); + } + } + } + fn shared_effective_priority( - subscribers: &HashMap, + subscribers: &HashMap, kind: ConnectionKind, ) -> Option { subscribers.values().filter(|subscriber| subscriber.kind == kind).map(|subscriber| subscriber.priority).min() } - fn shared_effective_kind(subscribers: &HashMap) -> ConnectionKind { + fn shared_effective_kind(subscribers: &HashMap) -> ConnectionKind { if subscribers.values().all(|subscriber| subscriber.kind == ConnectionKind::Soft) { ConnectionKind::Soft } else { @@ -177,16 +510,42 @@ impl ActiveProviderManager { let grace_period_options = Self::get_grace_options(cfg); let inputs = Self::get_config_inputs(cfg); Self { - providers: ProviderLineupManager::new(inputs, grace_period_options, event_manager), - connections: RwLock::new(Connections::default()), - reservations: RwLock::new(HashMap::new()), - next_allocation_id: AtomicU64::new(1), + core: Arc::new(ActiveProviderManagerCore { + capacity_transition: std::sync::Mutex::new(()), + is_shutting_down: AtomicBool::new(false), + shutdown_token: CancellationToken::new(), + providers: ProviderLineupManager::new(inputs, grace_period_options, event_manager), + connections: std::sync::RwLock::new(Connections::default()), + leases: std::sync::RwLock::new(ProviderLeaseTable::default()), + next_allocation_id: AtomicU64::new(1), + }), shared_stream_manager: OnceLock::new(), } } - pub fn set_shared_stream_manager(&self, manager: Arc) { - let _ = self.shared_stream_manager.set(manager); + pub fn set_shared_stream_manager(&self, manager: &Arc) { + let _ = self.shared_stream_manager.set(Arc::downgrade(manager)); + } + + /// Closes provider admission and releases all physical allocations and leases. + pub fn shutdown(&self) { + self.shutdown_token.cancel(); + self.is_shutting_down.store(true, Ordering::Release); + let _transition = self.lock_capacity_transition(); + let connections = std::mem::take(&mut *self.write_connections()); + self.write_leases().clear(); + + for info in connections.single.into_values() { + info.cancel_token.cancel(); + info.allocation.release(); + } + for shared in connections.shared.by_key.into_values() { + if let Some(cancel_token) = shared.cancel_token { + cancel_token.cancel(); + } + shared.allocation.release(); + } + self.providers.reconcile_connections(HashMap::new()); } fn get_config_inputs(cfg: &AppConfig) -> Vec> { @@ -195,24 +554,23 @@ impl ActiveProviderManager { fn get_grace_options(cfg: &AppConfig) -> GracePeriodOptions { cfg.config.load().get_grace_options() } - pub async fn update_config(&self, cfg: &AppConfig) { + pub fn update_config(&self, cfg: &AppConfig) { let grace_period_options = Self::get_grace_options(cfg); let inputs = Self::get_config_inputs(cfg); self.providers.update_config(inputs, &grace_period_options); - self.reconcile_connections().await; + self.reconcile_connections(); } - pub async fn reconcile_connections(&self) { + pub fn reconcile_connections(&self) { + let _transition = self.lock_capacity_transition(); let mut counts = HashMap::, usize>::new(); { - let connections = self.connections.read().await; + let connections = self.read_connections(); // Single connections - for per_addr in connections.single.values() { - for info in per_addr.values() { - if let Some(name) = info.allocation.get_provider_name() { - *counts.entry(name).or_insert(0) += 1; - } + for info in connections.single.values() { + if let Some(name) = info.allocation.get_provider_name() { + *counts.entry(name).or_insert(0) += 1; } } @@ -224,92 +582,105 @@ impl ActiveProviderManager { } } - self.providers.reconcile_connections(counts).await; + self.providers.reconcile_connections(counts); } - fn prune_expired_reservations(reservations: &mut ProviderReservations) { - let now = TokioInstant::now(); - reservations.retain(|_, provider_reservations| { - provider_reservations.retain(|_, reservation| reservation.expires_at > now); - !provider_reservations.is_empty() - }); - } + fn prune_expired_leases(leases: &mut ProviderLeaseTable) { leases.prune(TokioInstant::now()); } - async fn has_foreign_reservation(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { + fn has_foreign_reservation(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { if !self.providers.reservation_blocks_other_sessions(provider_name) { return false; } - let mut reservations = self.reservations.write().await; - Self::prune_expired_reservations(&mut reservations); - reservations.get(provider_name).is_some_and(|provider_reservations| { - session_owner.map_or(!provider_reservations.is_empty(), |owner| { - provider_reservations.keys().any(|reserved_owner| reserved_owner.as_ref() != owner) + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.has_foreign_reserved_lease(provider_name, session_owner) + } + + fn get_reserved_provider_for_owner(&self, input_name: &Arc, session_owner: &str) -> Option> { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let provider_name = leases.provider_for_owner(session_owner)?; + self.providers.is_provider_for_input(&provider_name, input_name).then_some(provider_name) + } + + /// True when `session_owner` already backs a live allocation on `provider_name`, + /// without materialising the full owner set for a membership test. + fn has_active_owner_for_provider(&self, provider_name: &Arc, session_owner: &str) -> bool { + let connections = self.read_connections(); + connections.by_owner.get(session_owner).is_some_and(|alloc_ids| { + alloc_ids.iter().any(|id| { + if let Some(info) = connections.single.get(id) { + info.allocation.get_provider_name().as_ref() == Some(provider_name) + } else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) { + connections + .shared + .by_key + .get(key) + .is_some_and(|shared| shared.allocation.get_provider_name().as_ref() == Some(provider_name)) + } else { + false + } }) }) } - async fn get_reserved_provider_for_owner(&self, input_name: &Arc, session_owner: &str) -> Option> { - let mut reservations = self.reservations.write().await; - Self::prune_expired_reservations(&mut reservations); - reservations.iter().find_map(|(provider_name, provider_reservations)| { - (provider_reservations.contains_key(session_owner) - && self.providers.is_provider_for_input(provider_name, input_name)) - .then(|| provider_name.clone()) - }) - } - - async fn active_reservation_owners(&self, provider_name: &Arc) -> HashSet> { - let connections = self.connections.read().await; + fn active_reservation_owners(&self, provider_name: &Arc) -> HashSet> { + let connections = self.read_connections(); let mut owners = HashSet::new(); - for per_addr in connections.single.values() { - for info in per_addr.values() { + if let Some(alloc_ids) = connections.by_provider.get(provider_name) { + for id in alloc_ids { + if let Some(info) = connections.single.get(id) { + if let Some(owner) = info.session_owner.as_ref() { + owners.insert(Arc::clone(owner)); + } + } else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) { + if let Some(shared) = connections.shared.by_key.get(key) { + if let Some(owner) = shared.session_owner.as_ref() { + owners.insert(Arc::clone(owner)); + } + } + } + } + } else { + for info in connections.single.values() { if info.allocation.get_provider_name().as_ref() == Some(provider_name) { if let Some(owner) = info.session_owner.as_ref() { owners.insert(Arc::clone(owner)); } } } - } - for shared in connections.shared.by_key.values() { - if shared.allocation.get_provider_name().as_ref() == Some(provider_name) { - if let Some(owner) = shared.session_owner.as_ref() { - owners.insert(Arc::clone(owner)); + for shared in connections.shared.by_key.values() { + if shared.allocation.get_provider_name().as_ref() == Some(provider_name) { + if let Some(owner) = shared.session_owner.as_ref() { + owners.insert(Arc::clone(owner)); + } } } } owners } - async fn reservation_capacity_usage( + fn reservation_capacity_usage( &self, provider_name: &Arc, session_owner: Option<&str>, ) -> Option<(usize, usize, usize)> { - let (current_connections, max_connections) = self.providers.provider_capacity(provider_name).await?; + let (current_connections, max_connections) = self.providers.provider_capacity(provider_name)?; if max_connections == 0 { return Some((current_connections, max_connections, 0)); } - - let active_owners = self.active_reservation_owners(provider_name).await; - let mut reservations = self.reservations.write().await; - Self::prune_expired_reservations(&mut reservations); - let idle_foreign_reservations = reservations.get(provider_name).map_or(0, |provider_reservations| { - provider_reservations - .keys() - .filter(|reserved_owner| { - let reserved_owner = reserved_owner.as_ref(); - session_owner != Some(reserved_owner) && !active_owners.contains(reserved_owner) - }) - .count() - }); + let counted_owners = self.active_reservation_owners(provider_name); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let idle_foreign_reservations = leases.foreign_reserved_slots(provider_name, session_owner, &counted_owners); Some((current_connections, max_connections, idle_foreign_reservations)) } /// Returns true when the next allocation would consume a slot kept for an idle reservation. - async fn reserved_capacity_blocks_next(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { + fn reserved_capacity_blocks_next(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { let Some((current_connections, max_connections, idle_foreign_reservations)) = - self.reservation_capacity_usage(provider_name, session_owner).await + self.reservation_capacity_usage(provider_name, session_owner) else { return true; }; @@ -319,9 +690,9 @@ impl ActiveProviderManager { } /// Returns true when an already-counted candidate allocation consumed a slot kept for an idle reservation. - async fn exceeds_reserved_capacity(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { + fn exceeds_reserved_capacity(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { let Some((current_connections, max_connections, idle_foreign_reservations)) = - self.reservation_capacity_usage(provider_name, session_owner).await + self.reservation_capacity_usage(provider_name, session_owner) else { return true; }; @@ -330,89 +701,370 @@ impl ActiveProviderManager { && current_connections.saturating_add(idle_foreign_reservations) > max_connections } - async fn reserved_provider_names_for_other( + /// Explains a reservation skip with the authoritative slot breakdown instead of + /// only the transport socket, so the decision can be audited from the logs. + fn log_reserved_capacity_skip(&self, provider_name: &Arc, params: &AcquireProviderParams<'_>) { + if !log::log_enabled!(log::Level::Debug) { + return; + } + let Some((current_connections, max_connections, foreign_reserved)) = + self.reservation_capacity_usage(provider_name, params.session_owner()) + else { + debug_if_enabled!( + "Skipping reserved provider {} (reason={}, capacity=unknown, peer_addr={}, request_id={}, playback_kind={})", + sanitize_sensitive_info(provider_name), + PlaybackSelectionReason::ReservedCapacity, + sanitize_sensitive_info(¶ms.addr.to_string()), + params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()), + params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) + ); + return; + }; + let usage = self.provider_lease_usage(provider_name); + debug_if_enabled!( + "Skipping reserved provider {} (reason={}, current={}, max={}, foreign_reserved={}, active_slots={}, starting_slots={}, idle_slots={}, peer_addr={}, request_id={}, playback_kind={})", + sanitize_sensitive_info(provider_name), + PlaybackSelectionReason::ReservedCapacity, + current_connections, + max_connections, + foreign_reserved, + usage.active, + usage.starting, + usage.idle, + sanitize_sensitive_info(¶ms.addr.to_string()), + params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()), + params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) + ); + } + + fn reserved_provider_names_for_other( &self, input_name: &Arc, session_owner: Option<&str>, ) -> HashSet> { let mut reserved = HashSet::new(); for provider_name in self.providers.provider_names_for_input(input_name) { - if self.has_foreign_reservation(&provider_name, session_owner).await { + if self.has_foreign_reservation(&provider_name, session_owner) { reserved.insert(provider_name); } } reserved } - pub async fn refresh_provider_reservation(&self, provider_name: &Arc, session_owner: &str, ttl_secs: u64) { - let mut reservations = self.reservations.write().await; - Self::prune_expired_reservations(&mut reservations); - reservations.retain(|_, provider_reservations| { - provider_reservations.remove(session_owner); - !provider_reservations.is_empty() - }); - if ttl_secs == 0 { + pub fn refresh_provider_reservation(&self, provider_name: &Arc, session_owner: &str, ttl_secs: u64) { + self.refresh_identified_provider_reservation(provider_name, session_owner, PlaybackKind::LiveHls, ttl_secs); + } + + /// Recreates or renews a provider reservation for an owner with an explicit kind. + /// + /// Unlike [`Self::refresh_adaptive_playback_lease`] this recreates the lease when it + /// no longer exists, which is what a failed preemption needs to restore a victim + /// reservation that [`Self::clear_identified_provider_reservation`] already removed. + pub fn refresh_identified_provider_reservation( + &self, + provider_name: &Arc, + session_owner: &str, + kind: PlaybackKind, + ttl_secs: u64, + ) { + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { return; } - reservations.entry(provider_name.clone()).or_default().insert( - Arc::from(session_owner), - ProviderReservation { expires_at: TokioInstant::now() + Duration::from_secs(ttl_secs) }, + let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + if ttl_secs == 0 { + leases.release_owner(session_owner); + return; + } + if leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl).is_none() { + let req_id = PlaybackRequestId::next(); + let _ = leases.begin_owner(session_owner, provider_name, kind, req_id); + leases.renew_identified_owner(session_owner, provider_name, kind, req_id, granted_ttl); + } + } + + /// Renews the lease of an adaptive (HLS/DASH/Catchup) playback. + /// + /// Adaptive playback consists of many short requests, so the lease must stay + /// reconnect-capable: between two segment requests the slot is kept as an idle + /// lease on the same provider instead of being released and reselected. + pub fn refresh_adaptive_playback_lease( + &self, + provider_name: &Arc, + session_owner: &str, + kind: PlaybackKind, + ttl_secs: u64, + ) { + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return; + } + let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl); + } + + /// Renews the lease of a playback. A zero TTL clears it. + pub fn refresh_playback_lease(&self, provider_name: &Arc, lease_ref: &PlaybackLeaseRef<'_>, ttl_secs: u64) { + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return; + } + let granted_ttl = self.renewal_granted_ttl(provider_name, lease_ref.owner, ttl_secs); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + if ttl_secs == 0 { + leases.release_identified_owner(lease_ref.owner, lease_ref.request_id); + return; + } + leases.renew_identified_owner( + lease_ref.owner, + provider_name, + lease_ref.kind, + lease_ref.request_id, + granted_ttl, ); } - pub async fn clear_provider_reservation(&self, session_owner: &str) { - let mut reservations = self.reservations.write().await; - Self::prune_expired_reservations(&mut reservations); - reservations.retain(|_, provider_reservations| { - provider_reservations.remove(session_owner); - !provider_reservations.is_empty() - }); + /// The reconnect window a renewal may actually grant for `owner`. + /// + /// A lease that currently holds no reservation right must not gain one through a + /// mere activity refresh: the grant is re-checked against provider capacity with + /// the same admission rule as a confirmation, so a previously denied reservation + /// cannot be silently restored by a later refresh. + fn renewal_granted_ttl(&self, provider_name: &Arc, owner: &str, requested_ttl_secs: u64) -> u64 { + if requested_ttl_secs == 0 { + return 0; + } + let already_reserves = { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.lease_of_owner(owner).is_some_and(|lease| lease.state.is_confirmed() && lease.idle_ttl_secs > 0) + }; + if already_reserves || self.confirmation_reserve_allowed(provider_name, owner) { + requested_ttl_secs + } else { + 0 + } } - async fn acquire_exact_connection_inner( + pub fn clear_provider_reservation(&self, session_owner: &str) { + let _transition = self.lock_capacity_transition(); + let mut leases = self.write_leases(); + leases.release_owner(session_owner); + } + + pub fn clear_identified_provider_reservation( + &self, + session_owner: &str, + provider_name: &Arc, + binding_tag: Option, + ) { + // A delayed clear without the exact binding tag has no delete right: it must + // not fall back to an owner-wide release and erase a successor lease. + let Some(binding_tag) = binding_tag else { + return; + }; + let _transition = self.lock_capacity_transition(); + let mut leases = self.write_leases(); + leases.release_matching_owner(session_owner, provider_name, binding_tag); + } + + /// Confirms real media activity for a playback lease. Only a confirmed lease may + /// outlive its request as a reconnect slot. + pub fn confirm_playback_activity(&self, owner: &str) -> Option { self.confirm_owner(owner, None) } + + /// Confirms real media activity for a specific request ID of a playback lease. + pub fn confirm_identified_playback_activity( + &self, + owner: &str, + request_id: PlaybackRequestId, + ) -> Option { + self.confirm_owner(owner, Some(request_id)) + } + + /// Records media activity under the capacity transition. The confirmation only + /// grants a reservation right when the lease already backs a live allocation or a + /// free slot remains; otherwise the media bytes are recorded without a reservation + /// so a late cache confirmation cannot over-commit a provider at its limit. + fn confirm_owner(&self, owner: &str, request_id: Option) -> Option { + let _transition = self.lock_capacity_transition(); + let (provider_name, wants_reservation) = { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let provider_name = leases.provider_for_owner(owner)?; + let wants_reservation = leases.lease_of_owner(owner).is_some_and(|lease| lease.idle_ttl_secs > 0); + (provider_name, wants_reservation) + }; + let reserve = !wants_reservation || self.confirmation_reserve_allowed(&provider_name, owner); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + match (request_id, reserve) { + (Some(request_id), true) => leases.confirm_identified_owner(owner, request_id), + (Some(request_id), false) => leases.confirm_identified_owner_without_reservation(owner, request_id), + (None, true) => leases.confirm_owner(owner), + (None, false) => leases.confirm_owner_without_reservation(owner), + } + } + + /// True when confirming `session_owner`'s lease may reserve a provider slot: the + /// lease either already backs a live allocation, or a free slot remains below the + /// configured maximum after foreign reservations are counted. + fn confirmation_reserve_allowed(&self, provider_name: &Arc, session_owner: &str) -> bool { + if self.has_active_owner_for_provider(provider_name, session_owner) { + return true; + } + let Some((current, max, foreign)) = self.reservation_capacity_usage(provider_name, Some(session_owner)) else { + return false; + }; + max == 0 || current.saturating_add(foreign) < max + } + + /// Ends a playback request and applies the outcome-specific lease policy. + /// + /// `idle_ttl_secs` overrides the reconnect window stored on the lease; `None` + /// keeps the window the endpoint configured when it renewed the lease. + pub fn finish_playback_request(&self, owner: &str, outcome: PlaybackRequestOutcome, idle_ttl_secs: Option) { + self.finish_playback_request_inner(owner, None, outcome, idle_ttl_secs); + } + + pub fn finish_identified_playback_request( + &self, + owner: &str, + request_id: PlaybackRequestId, + outcome: PlaybackRequestOutcome, + ) { + self.finish_playback_request_inner(owner, Some(request_id), outcome, None); + } + + fn finish_playback_request_inner( + &self, + owner: &str, + request_id: Option, + outcome: PlaybackRequestOutcome, + idle_ttl_secs: Option, + ) { + let _transition = self.lock_capacity_transition(); + // A delayed completion must not end a newer request of this playback. + // Only active connections from a DIFFERENT request ID (or shared streams) prevent lease finish. + let connections = self.read_connections(); + let has_other_active_connection = connections.by_owner.get(owner).is_some_and(|alloc_ids| { + alloc_ids.iter().any(|id| { + connections + .single + .get(id) + .is_some_and(|info| request_id.is_none() || info.playback_request_id != request_id) + || connections.shared.shared_by_allocation_id.contains_key(id) + }) + }); + if has_other_active_connection { + if let Some(req_id) = request_id { + let mut leases = self.write_leases(); + leases.detach_identified_request(owner, req_id); + } + return; + } + drop(connections); + let mut leases = self.write_leases(); + if let Some(req_id) = request_id { + leases.finish_identified_owner(owner, req_id, outcome, idle_ttl_secs); + } else { + leases.finish_owner(owner, outcome, idle_ttl_secs); + } + } + + /// Snapshot of the logical slots a provider holds, for logs and UI metrics. + pub fn provider_lease_usage(&self, provider_name: &Arc) -> ProviderLeaseUsage { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.usage(provider_name) + } + + /// Removes every expired starting/idle lease. Runs from the session GC task. + pub fn prune_expired_leases_now(&self) { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + } + + /// The provider lease's binding tag for an owner. The HLS layer captures this + /// from the acquired handle (via `ProviderHandle::binding_tag`) and passes it + /// back to [`Self::clear_identified_provider_reservation`] so a stale detach + /// cannot delete a successor lease on the same account. + pub fn binding_tag_for_owner(&self, owner: &str) -> Option { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.lease_of_owner(owner).map(|lease| ProviderBindingTag::new(lease.id, lease.binding_generation)) + } + + fn acquire_exact_connection_inner_no_preempt( &self, provider_name: &Arc, allow_grace: bool, params: &AcquireProviderParams<'_>, ) -> Option { - let allocation = self.providers.acquire_exact_connection_with_grace_override(provider_name, allow_grace).await; - if matches!(allocation, ProviderAllocation::Exhausted) { + let mut allocation = ProviderAllocationGuard::new( + self.providers.acquire_exact_connection_with_grace_override(provider_name, allow_grace), + ); + if matches!(allocation.allocation(), ProviderAllocation::Exhausted) { return None; } - if self.exceeds_reserved_capacity(provider_name, params.session_owner).await { - allocation.release().await; + if self.exceeds_reserved_capacity(provider_name, params.session_owner()) { return None; } - self.register_allocation(allocation, params).await + Some(self.register_allocation(allocation.take(), params)) } - async fn finalize_lineup_allocation( + fn acquire_exact_connection_inner( + &self, + provider_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, params) { + return Some(handle); + } + if let Some(preempted_alloc) = self.try_preempt_connection( + provider_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) { + return Some(self.register_allocation(preempted_alloc, params)); + } + None + } + + fn finalize_lineup_allocation( &self, input_name: &Arc, allow_grace: bool, - allocation: ProviderAllocation, + mut allocation: ProviderAllocationGuard, params: &AcquireProviderParams<'_>, - ) -> Option { - if matches!(&allocation, ProviderAllocation::GracePeriod(_)) - && self.evict_lower_priority_on_input(input_name, params.priority, params.kind, params.session_owner).await + ) -> ProviderHandle { + if matches!(allocation.allocation(), ProviderAllocation::GracePeriod(_)) + && self.evict_lower_priority_on_input(input_name, params.priority, params.kind, params.session_owner()) { let evicted_on_same = - !self.providers.is_over_limit(&allocation.get_provider_name().unwrap_or_default()).await; + !self.providers.is_over_limit(&allocation.allocation().get_provider_name().unwrap_or_default()); if !evicted_on_same { - let new_alloc = self.providers.acquire_connection_with_grace_override(input_name, allow_grace).await; - if !matches!(new_alloc, ProviderAllocation::Exhausted) { - if let Some(provider_name) = new_alloc.get_provider_name() { - if !self.exceeds_reserved_capacity(&provider_name, params.session_owner).await { - allocation.release().await; - return self.register_allocation(new_alloc, params).await; + let mut new_alloc = ProviderAllocationGuard::new( + self.providers.acquire_connection_with_grace_override(input_name, allow_grace), + ); + if !matches!(new_alloc.allocation(), ProviderAllocation::Exhausted) { + if let Some(provider_name) = new_alloc.allocation().get_provider_name() { + if !self.exceeds_reserved_capacity(&provider_name, params.session_owner()) { + return self.register_allocation(new_alloc.take(), params); } } - new_alloc.release().await; } } } - self.register_allocation(allocation, params).await + self.register_allocation(allocation.take(), params) } fn select_victim_from_index( @@ -425,7 +1077,9 @@ impl ActiveProviderManager { let mut victim = None; for (prov_name, tree) in index { - if !self.providers.is_provider_for_input(prov_name, input_name) || reserved_providers.contains(prov_name) { + if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) + || reserved_providers.contains(prov_name) + { continue; } @@ -454,12 +1108,11 @@ impl ActiveProviderManager { candidate: PreemptionCandidate, ) -> bool { match candidate.0 { - PriorityOwner::Single(addr) => { - connections.single.get(&addr).and_then(|per_addr| per_addr.get(&candidate.1)).is_some_and(|info| { - info.kind == ConnectionKind::Normal - && info.allocation.get_provider_name().as_ref() == Some(provider_name) - }) - } + PriorityOwner::Single(alloc_id) => connections.single.get(&alloc_id).is_some_and(|info| { + info.kind == ConnectionKind::Normal + && info.lifecycle != ConnectionLifecycle::Closing + && info.allocation.get_provider_name().as_ref() == Some(provider_name) + }), PriorityOwner::Shared(shared_id) => connections .shared .shared_by_allocation_id @@ -495,7 +1148,7 @@ impl ActiveProviderManager { let mut victim = None; for (prov_name, tree) in &connections.priority_index { - if !self.providers.is_provider_for_input(prov_name, input_name) + if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) || reserved_providers.contains(prov_name) { continue; @@ -527,15 +1180,15 @@ impl ActiveProviderManager { } } - async fn acquire_connection_inner( + fn acquire_connection_inner_no_preempt( &self, provider_or_input_name: &Arc, allow_grace: bool, params: &AcquireProviderParams<'_>, ) -> Option { - if let Some(owner) = params.session_owner { - if let Some(reserved_provider) = self.get_reserved_provider_for_owner(provider_or_input_name, owner).await { - return self.acquire_exact_connection_inner(&reserved_provider, allow_grace, params).await; + if let Some(owner) = params.session_owner() { + if let Some(reserved_provider) = self.get_reserved_provider_for_owner(provider_or_input_name, owner) { + return self.acquire_exact_connection_inner_no_preempt(&reserved_provider, allow_grace, params); } } @@ -543,77 +1196,123 @@ impl ActiveProviderManager { let attempts = candidate_count.max(1); let mut skipped_reserved = HashSet::new(); for _ in 0..attempts { - let allocation = self - .providers - .acquire_connection_with_grace_override_excluding( + let allocation = + ProviderAllocationGuard::new(self.providers.acquire_connection_with_grace_override_excluding( provider_or_input_name, allow_grace, &skipped_reserved, - ) - .await; - if matches!(allocation, ProviderAllocation::Exhausted) { + )); + if matches!(allocation.allocation(), ProviderAllocation::Exhausted) { break; } - if let Some(provider_name) = allocation.get_provider_name() { - if self.exceeds_reserved_capacity(&provider_name, params.session_owner).await { - debug_if_enabled!( - "Skipping reserved provider {} for {}", - sanitize_sensitive_info(&provider_name), - sanitize_sensitive_info(¶ms.addr.to_string()) - ); + if let Some(provider_name) = allocation.allocation().get_provider_name() { + if self.exceeds_reserved_capacity(&provider_name, params.session_owner()) { + self.log_reserved_capacity_skip(&provider_name, params); skipped_reserved.insert(provider_name); - allocation.release().await; if skipped_reserved.len() >= attempts { break; } continue; } } - return self.finalize_lineup_allocation(provider_or_input_name, allow_grace, allocation, params).await; - } - - if let Some(preempted_alloc) = self - .try_preempt_connection( - provider_or_input_name, - params.priority, - allow_grace, - params.kind, - params.session_owner, - ) - .await - { - return self.register_allocation(preempted_alloc, params).await; + return Some(self.finalize_lineup_allocation(provider_or_input_name, allow_grace, allocation, params)); } None } - async fn register_allocation( + fn acquire_connection_inner( + &self, + provider_or_input_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_connection_inner_no_preempt(provider_or_input_name, allow_grace, params) { + return Some(handle); + } + + if let Some(preempted_alloc) = self.try_preempt_connection( + provider_or_input_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) { + return Some(self.register_allocation(preempted_alloc, params)); + } + + None + } + + fn register_allocation( &self, allocation: ProviderAllocation, params: &AcquireProviderParams<'_>, - ) -> Option { - let AcquireProviderParams { addr, priority, kind, session_owner } = *params; + ) -> ProviderHandle { + let AcquireProviderParams { addr, priority, kind, lease } = *params; + let session_owner = lease.map(|lease| lease.owner); let provider_name = allocation.get_provider_name().unwrap_or_default(); let allocation_id = self.next_allocation_id.fetch_add(1, Ordering::Relaxed); let cancel_token = CancellationToken::new(); + let completion_token = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8)); let now = Instant::now(); let is_unlimited = allocation.is_unlimited_provider(); - let mut connections = self.connections.write().await; - let per_addr = connections.single.entry(*addr).or_default(); + // Claim the playback's capacity slot lease in its own lock scope so the + // connection and lease locks are never nested. The lease starts unconfirmed; + // it only reserves capacity against other playbacks once real media activity + // confirms it, which is what keeps abandoned manifest starts from blocking. + let binding_tag = if let Some(lease) = lease { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let id = leases.begin_owner(lease.owner, &provider_name, lease.kind, lease.request_id); + let generation = leases.lease(id).map_or(1, |lease| lease.binding_generation); + debug_if_enabled!( + "Playback lease began: provider={} owner={} kind={} request_id={} lease_id={} generation={} state=starting", + sanitize_sensitive_info(&provider_name), + sanitize_sensitive_info(lease.owner), + lease.kind, + lease.request_id, + id, + generation + ); + Some(ProviderBindingTag::new(id, generation)) + } else { + None + }; - per_addr.insert( + let session_owner_arc = session_owner.map(Arc::from); + let mut connections = self.write_connections(); + connections.single.insert( allocation_id, ActiveConnectionInfo { + allocation_id, + client_addr: *addr, allocation: allocation.clone(), cancel_token: cancel_token.clone(), + completion_token: completion_token.clone(), + close_reason: Arc::clone(&close_reason), + lifecycle: ConnectionLifecycle::Active, + has_body_owner: false, + reaper_spawned: false, + open_generation: 0, created_at: now, priority, kind, - session_owner: session_owner.map(Arc::from), + session_owner: session_owner_arc.clone(), + playback_request_id: lease.map(|lease| lease.request_id), }, ); + connections.single_by_addr.entry(*addr).or_default().insert(allocation_id); + Self::index_owner(&mut connections, allocation_id, session_owner_arc.as_ref()); // Unlimited providers are not subject to preemption, so we deliberately // skip populating the by_provider / priority_index / soft_priority_index @@ -623,39 +1322,47 @@ impl ActiveProviderManager { // even though it cannot be exhausted, contradicting the configured // `max_connections: 0` semantics. if !is_unlimited { - connections.by_provider.entry(provider_name.clone()).or_default().insert((*addr, allocation_id)); + connections.by_provider.entry(provider_name.clone()).or_default().insert(allocation_id); Self::upsert_priority_entry( &mut connections, &provider_name, (priority, Reverse(now), allocation_id), - PriorityOwner::Single(*addr), + PriorityOwner::Single(allocation_id), kind, ); } debug_if_enabled!( - "Added provider connection {provider_name:?} for {} (prio={}, kind={kind:?}, unlimited={is_unlimited})", - sanitize_sensitive_info(&addr.to_string()), - priority + "Added provider connection {provider_name:?} (reason={}, prio={priority}, kind={kind:?}, unlimited={is_unlimited}, allocation_id={allocation_id}, lease_id={}, request_id={}, playback_kind={}, peer_addr={})", + PlaybackSelectionReason::NewPriorityAllocation, + binding_tag.map_or_else(String::new, |tag| tag.lease_id.to_string()), + lease.map_or_else(String::new, |lease| lease.request_id.to_string()), + lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()), + sanitize_sensitive_info(&addr.to_string()) ); - Some(ProviderHandle::new(*addr, allocation_id, allocation, Some(cancel_token))) + let mut handle = ProviderHandle::new(*addr, allocation_id, allocation, Some(cancel_token)); + handle.completion_token = Some(completion_token); + handle.close_reason = close_reason; + handle.playback_request_id = lease.map(|lease| lease.request_id); + handle.binding_tag = binding_tag; + handle } #[allow(clippy::too_many_lines)] /// Evict a single lower-priority connection across the entire input lineup /// (all provider aliases). Used when a `GracePeriod` allocation was granted. /// Returns true if a victim was successfully evicted. - async fn evict_lower_priority_on_input( + fn evict_lower_priority_on_input( &self, input_name: &Arc, new_priority: i8, kind_needed: ConnectionKind, session_owner: Option<&str>, ) -> bool { - let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner).await; + let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); let victim = { - let connections = self.connections.read().await; + let connections = self.read_connections(); self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) }; @@ -671,7 +1378,7 @@ impl ActiveProviderManager { ); let released = { - let mut connections = self.connections.write().await; + let mut connections = self.write_connections(); let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else { return false; }; @@ -687,12 +1394,13 @@ impl ActiveProviderManager { if let Some(shared) = connections.shared.by_key.remove(&key) { connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); - for shared_addr in shared.connections.keys() { - connections.shared.key_by_addr.remove(shared_addr); + Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); } if let Some(name) = shared.allocation.get_provider_name() { if let Some(list) = connections.by_provider.get_mut(&name) { - list.retain(|(_, i)| *i != shared.allocation_id); + list.remove(&shared.allocation_id); } Self::remove_priority_entry( &mut connections, @@ -710,72 +1418,250 @@ impl ActiveProviderManager { if let Some(token) = cancel_token { token.cancel(); } - allocation.release().await; - // Stop the shared stream broadcast task to match the released capacity. - // Without this, the broadcast keeps running and consuming a provider slot - // that was already freed by allocation.release(). - if let Some(ssm) = self.shared_stream_manager.get() { - ssm.teardown_preempted_stream(&stream_url).await; + if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { + tokio::spawn(async move { + ssm.teardown_preempted_stream(&stream_url, alloc_id).await; + allocation.release(); + }); + return false; } + allocation.release(); } } - PriorityOwner::Single(addr) => { - debug_if_enabled!( - "Grace-evicting single connection from {} (prio={}) on input {} for higher priority request (prio={})", - sanitize_sensitive_info(&addr.to_string()), - v_prio, - sanitize_sensitive_info(input_name), - new_priority - ); - - let removed_info = { - let mut connections = self.connections.write().await; - let mut removed_info = None; - let mut removed_provider_name = None; - let mut remove_addr_entry = false; - if let Some(per_addr) = connections.single.get_mut(&addr) { - if let Some(info) = per_addr.get(&alloc_id) { - if info.priority != v_prio || info.created_at != victim_created_at { - return false; + PriorityOwner::Single(victim_alloc_id) => { + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get(&victim_alloc_id) { + if info.priority != v_prio || info.created_at != victim_created_at { + return false; + } + } + if let Some(info) = connections.single.get_mut(&victim_alloc_id) { + debug_if_enabled!( + "Grace-evicting single connection from {} (prio={}) on input {} for higher priority request (prio={})", + sanitize_sensitive_info(&info.client_addr.to_string()), + v_prio, + sanitize_sensitive_info(input_name), + new_priority + ); + info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); + } + let action = ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); + if let ReleaseAction::Wait(alloc_id, gen, completion_token) = action { + let shutdown_token = self.shutdown_token.clone(); + let core = Arc::clone(&self.core); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); } } - if let Some(info) = per_addr.remove(&alloc_id) { - removed_provider_name = info.allocation.get_provider_name(); - remove_addr_entry = per_addr.is_empty(); - removed_info = Some(info); - } - } - if remove_addr_entry { - connections.single.remove(&addr); - } - if let Some(ref name) = removed_provider_name { - if let Some(list) = connections.by_provider.get_mut(name) { - list.remove(&(addr, alloc_id)); - } - Self::remove_priority_entry( - &mut connections, - name, - &(v_prio, Reverse(victim_created_at), alloc_id), - removed_info.as_ref().map_or(ConnectionKind::Normal, |info| info.kind), - ); - } - removed_info - }; - - let Some(info) = removed_info else { + }); return false; - }; - // Preempted probes must stop immediately; they must not keep a custom stream alive. - info.cancel_token.cancel(); - info.allocation.release().await; + } } } true } + fn try_acquire_allocation_after_freed( + &self, + input_name: &Arc, + allow_grace: bool, + reserved_providers: &HashSet>, + session_owner: Option<&str>, + ) -> Option { + let attempts = self.providers.provider_names_for_input(input_name).len().max(1); + let mut excluded_providers = reserved_providers.clone(); + for _ in 0..attempts { + let allocation = self.providers.acquire_connection_with_grace_override_excluding( + input_name, + allow_grace, + &excluded_providers, + ); + if matches!(allocation, ProviderAllocation::Exhausted) { + break; + } + if let Some(provider_name) = allocation.get_provider_name() { + if self.exceeds_reserved_capacity(&provider_name, session_owner) { + excluded_providers.insert(provider_name); + allocation.release(); + continue; + } + } + return Some(allocation); + } + None + } + #[allow(clippy::too_many_lines)] - async fn try_preempt_connection( + fn try_preempt_connection_outcome( + &self, + input_name: &Arc, + new_priority: i8, + allow_grace: bool, + kind_needed: ConnectionKind, + session_owner: Option<&str>, + ) -> PreemptionOutcome { + let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); + let victim = { + let connections = self.read_connections(); + self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) + }; + + let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else { + return PreemptionOutcome::Exhausted; + }; + + match owner { + PriorityOwner::Shared(shared_id) => { + debug_if_enabled!( + "Preempting shared connection (allocation_id={shared_id}, prio={v_prio}) for higher priority request (prio={new_priority})" + ); + let released_shared_allocation = { + let mut connections = self.write_connections(); + let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else { + return PreemptionOutcome::Exhausted; + }; + + let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { + shared.allocation_id == alloc_id + && shared.priority == v_prio + && shared.created_at == victim_created_at + }); + if !still_match { + return PreemptionOutcome::Exhausted; + } + + if let Some(shared) = connections.shared.by_key.remove(&key) { + connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); + Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); + } + + if let Some(name) = shared.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&shared.allocation_id); + } + Self::remove_priority_entry( + &mut connections, + &name, + &(v_prio, Reverse(victim_created_at), alloc_id), + shared.kind, + ); + } + Some((key, shared.allocation, shared.cancel_token)) + } else { + None + } + }; + + let Some((stream_url, allocation, cancel_token)) = released_shared_allocation else { + return PreemptionOutcome::Exhausted; + }; + + if let Some(token) = cancel_token { + token.cancel(); + } + if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { + // Signal completion after teardown finishes so async acquirers can wait for the + // actual upstream shutdown rather than receiving Exhausted immediately. + let done_token = CancellationToken::new(); + let done_signal = done_token.clone(); + tokio::spawn(async move { + ssm.teardown_preempted_stream(&stream_url, alloc_id).await; + allocation.release(); + done_signal.cancel(); + }); + PreemptionOutcome::PendingCompletion(None, done_token) + } else { + allocation.release(); + if let Some(alloc) = self.try_acquire_allocation_after_freed( + input_name, + allow_grace, + &reserved_providers, + session_owner, + ) { + PreemptionOutcome::Acquired(alloc) + } else { + PreemptionOutcome::Exhausted + } + } + } + PriorityOwner::Single(victim_alloc_id) => { + let (action, completion_token) = { + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get(&victim_alloc_id) { + if info.priority != v_prio || info.created_at != victim_created_at { + return PreemptionOutcome::Exhausted; + } + } else { + return PreemptionOutcome::Exhausted; + } + if let Some(info) = connections.single.get_mut(&victim_alloc_id) { + debug_if_enabled!( + "Preempting single connection from {} (prio={v_prio}) for higher priority request (prio={new_priority})", + sanitize_sensitive_info(&info.client_addr.to_string()) + ); + info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); + } + let completion_token = + connections.single.get(&victim_alloc_id).map(|info| info.completion_token.clone()); + let action = + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); + (action, completion_token) + }; + + match action { + ReleaseAction::Wait(victim_id, gen, comp_token) => { + let shutdown_token = self.shutdown_token.clone(); + let core = Arc::clone(&self.core); + let reaper_token = comp_token.clone(); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = reaper_token.cancelled() => { + core.complete_release_with_generation(victim_id, Some(gen)); + } + } + }); + PreemptionOutcome::PendingCompletion(Some((victim_id, gen)), comp_token) + } + ReleaseAction::None => { + if let Some(token) = completion_token { + // Retrieve the current generation for the idempotent self-release path. + let victim_identity = { + let connections = self.read_connections(); + connections + .single + .get(&victim_alloc_id) + .map(|info| (victim_alloc_id, info.open_generation)) + }; + PreemptionOutcome::PendingCompletion(victim_identity, token) + } else { + PreemptionOutcome::Exhausted + } + } + ReleaseAction::Immediate => { + if let Some(alloc) = self.try_acquire_allocation_after_freed( + input_name, + allow_grace, + &reserved_providers, + session_owner, + ) { + PreemptionOutcome::Acquired(alloc) + } else { + PreemptionOutcome::Exhausted + } + } + } + } + } + } + + fn try_preempt_connection( &self, input_name: &Arc, new_priority: i8, @@ -783,151 +1669,13 @@ impl ActiveProviderManager { kind_needed: ConnectionKind, session_owner: Option<&str>, ) -> Option { - let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner).await; - let victim = { - let connections = self.connections.read().await; - self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) - }; - - if let Some((owner, alloc_id, v_prio, victim_created_at)) = victim { - match owner { - PriorityOwner::Shared(shared_id) => { - debug_if_enabled!( - "Preempting shared connection (allocation_id={shared_id}, prio={v_prio}) for higher priority request (prio={new_priority})" - ); - let released_shared_allocation = { - let mut connections = self.connections.write().await; - let key = connections.shared.shared_by_allocation_id.get(&shared_id).cloned()?; - - // Revalidate the selected shared victim under WRITE lock. - let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { - shared.allocation_id == alloc_id - && shared.priority == v_prio - && shared.created_at == victim_created_at - }); - if !still_match { - None - } else if let Some(shared) = connections.shared.by_key.remove(&key) { - connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); - for shared_addr in shared.connections.keys() { - connections.shared.key_by_addr.remove(shared_addr); - } - - if let Some(name) = shared.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.retain(|(_, i)| *i != shared.allocation_id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(v_prio, Reverse(victim_created_at), alloc_id), - shared.kind, - ); - } - Some((key, shared.allocation, shared.cancel_token)) - } else { - None - } - }; - let (stream_url, allocation, cancel_token) = released_shared_allocation?; - if let Some(token) = cancel_token { - token.cancel(); - } - allocation.release().await; - // Shared broadcast has to be torn down explicitly, otherwise the provider - // stream may continue after allocation counters were already released. - if let Some(ssm) = self.shared_stream_manager.get() { - ssm.teardown_preempted_stream(&stream_url).await; - } else { - error!( - "SharedStreamManager not initialised during preemption teardown for {}; \ - shared stream may linger after allocation release", - sanitize_sensitive_info(&stream_url) - ); - } - } - PriorityOwner::Single(addr) => { - debug_if_enabled!( - "Preempting single connection from {} (prio={v_prio}) for higher priority request (prio={new_priority})", - sanitize_sensitive_info(&addr.to_string()) - ); - - // Atomically remove the victim single connection and take ownership of the token. - // This guarantees only one concurrent preemptor can schedule delayed cancellation. - let removed_info = { - let mut connections = self.connections.write().await; - - let mut removed_info = None; - let mut removed_provider_name = None; - let mut remove_addr_entry = false; - if let Some(per_addr) = connections.single.get_mut(&addr) { - if let Some(info) = per_addr.get(&alloc_id) { - // Revalidate victim selection under write lock. - if info.priority != v_prio || info.created_at != victim_created_at { - return None; - } - } - if let Some(info) = per_addr.remove(&alloc_id) { - removed_provider_name = info.allocation.get_provider_name(); - remove_addr_entry = per_addr.is_empty(); - removed_info = Some(info); - } - } - - if remove_addr_entry { - connections.single.remove(&addr); - } - if let Some(name) = removed_provider_name { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&(addr, alloc_id)); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(v_prio, Reverse(victim_created_at), alloc_id), - removed_info.as_ref().map_or(ConnectionKind::Normal, |info| info.kind), - ); - } - removed_info - }; - - let Some(info) = removed_info else { - // Another preemptor already removed this victim. - return None; - }; - - // Preempted probes must stop immediately; they must not keep a custom stream alive. - info.cancel_token.cancel(); - info.allocation.release().await; - } - } - - // Now try acquire again preserving the original grace policy. - let attempts = self.providers.provider_names_for_input(input_name).len().max(1); - let mut excluded_providers = reserved_providers; - for _ in 0..attempts { - let allocation = self - .providers - .acquire_connection_with_grace_override_excluding(input_name, allow_grace, &excluded_providers) - .await; - if matches!(allocation, ProviderAllocation::Exhausted) { - break; - } - if let Some(provider_name) = allocation.get_provider_name() { - if self.exceeds_reserved_capacity(&provider_name, session_owner).await { - excluded_providers.insert(provider_name); - allocation.release().await; - continue; - } - } - return Some(allocation); - } + match self.try_preempt_connection_outcome(input_name, new_priority, allow_grace, kind_needed, session_owner) { + PreemptionOutcome::Acquired(alloc) => Some(alloc), + _ => None, } - - None } - pub async fn acquire_exact_connection_with_grace( + pub fn acquire_exact_connection_with_grace( &self, provider_name: &Arc, addr: &SocketAddr, @@ -936,10 +1684,9 @@ impl ActiveProviderManager { kind: ConnectionKind, ) -> Option { self.acquire_exact_connection_with_grace_for_session(provider_name, addr, allow_grace, priority, kind, None) - .await } - pub async fn acquire_exact_connection_with_grace_for_session( + pub fn acquire_exact_connection_with_grace_for_session( &self, provider_name: &Arc, addr: &SocketAddr, @@ -948,15 +1695,22 @@ impl ActiveProviderManager { kind: ConnectionKind, session_owner: Option<&str>, ) -> Option { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let lease = session_owner.map(PlaybackLeaseRef::for_session_owner); self.acquire_exact_connection_inner( provider_name, allow_grace, - &AcquireProviderParams { addr, priority, kind, session_owner }, + &AcquireProviderParams { addr, priority, kind, lease: lease.as_ref().copied() }, ) - .await } - pub async fn force_exact_acquire_connection( + pub fn force_exact_acquire_connection( &self, provider_name: &Arc, addr: &SocketAddr, @@ -964,27 +1718,22 @@ impl ActiveProviderManager { kind: ConnectionKind, ) -> Option { // Compatibility wrapper: keep the exact-provider behavior but do not over-allocate exhausted accounts. - self.acquire_exact_connection_with_grace(provider_name, addr, false, priority, kind).await + self.acquire_exact_connection_with_grace(provider_name, addr, false, priority, kind) } // Returns the next available provider connection - pub async fn acquire_connection( + pub fn acquire_connection( &self, input_name: &Arc, addr: &SocketAddr, priority: i8, kind: ConnectionKind, ) -> Option { - self.acquire_connection_inner( - input_name, - true, - &AcquireProviderParams { addr, priority, kind, session_owner: None }, - ) - .await + self.acquire_connection_inner(input_name, true, &AcquireProviderParams { addr, priority, kind, lease: None }) } /// Acquire a provider connection while explicitly controlling provider-side grace allocations. - pub async fn acquire_connection_with_grace( + pub fn acquire_connection_with_grace( &self, input_name: &Arc, addr: &SocketAddr, @@ -992,10 +1741,10 @@ impl ActiveProviderManager { priority: i8, kind: ConnectionKind, ) -> Option { - self.acquire_connection_with_grace_for_session(input_name, addr, allow_grace, priority, kind, None).await + self.acquire_connection_with_grace_for_session(input_name, addr, allow_grace, priority, kind, None) } - pub async fn acquire_connection_with_grace_for_session( + pub fn acquire_connection_with_grace_for_session( &self, input_name: &Arc, addr: &SocketAddr, @@ -1004,40 +1753,186 @@ impl ActiveProviderManager { kind: ConnectionKind, session_owner: Option<&str>, ) -> Option { - self.acquire_connection_inner( + self.acquire_connection_with_lease_for_session( input_name, + addr, allow_grace, - &AcquireProviderParams { addr, priority, kind, session_owner }, + priority, + kind, + session_owner.map(PlaybackLeaseRef::for_session_owner), ) - .await + } + + /// Lineup acquisition with an explicit playback lease identity. + pub fn acquire_connection_with_lease_for_session( + &self, + input_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + self.acquire_connection_inner(input_name, allow_grace, &AcquireProviderParams { addr, priority, kind, lease }) + } + + /// Lineup acquisition with an explicit playback lease identity, awaiting preemption completion if needed. + pub async fn acquire_connection_with_lease_for_session_await( + &self, + input_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let params = AcquireProviderParams { addr, priority, kind, lease }; + let outcome = { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) { + return Some(handle); + } + self.try_preempt_connection_outcome( + input_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) + }; + + match outcome { + PreemptionOutcome::Acquired(alloc) => { + let _transition = self.lock_capacity_transition(); + Some(self.register_allocation(alloc, ¶ms)) + } + PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => { + let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await; + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + // Idempotently free the victim's slot in case the background reaper has not yet run. + // complete_release_locked is a no-op if the slot was already freed or the generation + // no longer matches. + if let Some((victim_alloc_id, victim_gen)) = victim_identity { + let mut connections = self.write_connections(); + ActiveProviderManagerCore::complete_release_locked( + &mut connections, + victim_alloc_id, + Some(victim_gen), + ); + } + self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) + } + PreemptionOutcome::Exhausted => None, + } + } + + /// Exact-provider acquisition with an explicit playback lease identity. + pub fn acquire_exact_connection_with_lease_for_session( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let _transition = self.lock_capacity_transition(); + self.acquire_exact_connection_inner( + provider_name, + allow_grace, + &AcquireProviderParams { addr, priority, kind, lease }, + ) + } + + /// Exact-provider acquisition with an explicit playback lease identity, awaiting preemption completion if needed. + pub async fn acquire_exact_connection_with_lease_for_session_await( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let params = AcquireProviderParams { addr, priority, kind, lease }; + let outcome = { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) { + return Some(handle); + } + self.try_preempt_connection_outcome( + provider_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) + }; + + match outcome { + PreemptionOutcome::Acquired(alloc) => { + let _transition = self.lock_capacity_transition(); + Some(self.register_allocation(alloc, ¶ms)) + } + PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => { + let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await; + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if let Some((victim_alloc_id, victim_gen)) = victim_identity { + let mut connections = self.write_connections(); + ActiveProviderManagerCore::complete_release_locked( + &mut connections, + victim_alloc_id, + Some(victim_gen), + ); + } + self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) + } + PreemptionOutcome::Exhausted => None, + } } /// Acquire a provider connection for probe tasks with configurable priority. /// Probes never consume grace capacity. - pub async fn acquire_connection_for_probe(&self, input_name: &Arc, priority: i8) -> Option { + pub fn acquire_connection_for_probe(&self, input_name: &Arc, priority: i8) -> Option { self.acquire_connection_inner( input_name, false, - &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, session_owner: None }, + &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None }, ) - .await } /// Acquire a provider connection for background transfers (downloads/recordings). /// Transfers participate in the same provider priority/preemption model as normal /// streams, but they never consume grace capacity and wait externally on notifications. - pub async fn acquire_connection_for_download(&self, input_name: &Arc, priority: i8) -> Option { + pub fn acquire_connection_for_download(&self, input_name: &Arc, priority: i8) -> Option { self.acquire_connection_inner( input_name, false, - &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, session_owner: None }, + &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None }, ) - .await } // This method is used for redirects to cycle through the provider - pub async fn get_next_provider(&self, provider_name: &Arc) -> Option> { - self.providers.get_next_provider(provider_name).await + pub fn get_next_provider(&self, provider_name: &Arc) -> Option> { + self.providers.get_next_provider(provider_name) } pub fn find_provider_config(&self, provider_name: &Arc) -> Option> { @@ -1048,263 +1943,487 @@ impl ActiveProviderManager { self.providers.is_provider_for_input(provider_name.as_ref(), input_name.as_ref()) } - pub async fn is_provider_reserved_for_other_session( + pub fn is_provider_reserved_for_other_session( &self, provider_name: &Arc, session_owner: Option<&str>, ) -> bool { - self.reserved_capacity_blocks_next(provider_name, session_owner).await + self.reserved_capacity_blocks_next(provider_name, session_owner) } - pub async fn active_connections(&self) -> Option, usize>> { - self.providers.active_connections().await + pub fn active_connections(&self) -> Option, usize>> { self.providers.active_connections() } + + pub fn is_over_limit(&self, provider_name: &Arc) -> bool { self.providers.is_over_limit(provider_name) } + + pub fn is_exhausted(&self, provider_name: &Arc) -> bool { self.providers.is_exhausted(provider_name) } + + /// Finds every shared subscription carried by one transport socket. + /// + /// This is an explicit socket-wide transport action (connection close, kick), not + /// the normal playback cleanup path. A single shared subscription is released + /// through `release_shared_subscriber`, which cannot touch other clients that + /// happen to arrive through the same reverse-proxy socket. + fn release_shared_by_addr(&self, addr: &SocketAddr) -> Vec { + let connections = self.read_connections(); + connections + .shared + .by_key + .values() + .flat_map(|shared| { + shared.connections.iter().filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)) + }) + .collect() } - pub async fn is_over_limit(&self, provider_name: &Arc) -> bool { - self.providers.is_over_limit(provider_name).await - } + /// Releases exactly one shared subscription and rebalances the shared allocation. + /// + /// Returns the allocation to release when the last subscriber left. + fn release_shared_subscriber(&self, subscriber_id: SharedSubscriberId) -> Option { + let mut connections = self.write_connections(); - pub async fn is_exhausted(&self, provider_name: &Arc) -> bool { - self.providers.is_exhausted(provider_name).await - } + let key = connections.shared.key_by_subscriber.remove(&subscriber_id)?; + let mut shared = connections.shared.by_key.remove(&key)?; + shared.connections.remove(&subscriber_id); - #[allow(clippy::too_many_lines)] - pub async fn release_connection(&self, addr: &SocketAddr) { - // Single connection - all index updates in one lock scope - let single_allocations = { - let mut connections = self.connections.write().await; - if let Some(allocations) = connections.single.remove(addr) { - // Remove from by_provider and priority_index while still holding the lock. - // Skip these index updates for unlimited providers: they were never - // added to the preemption indices by `register_allocation`, so there - // is nothing to remove and the keys/values won't be present. - for (id, info) in &allocations { - if !info.allocation.is_unlimited_provider() { - if let Some(name) = info.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&(*addr, *id)); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(info.priority, Reverse(info.created_at), *id), - info.kind, - ); - } + let shared_is_unlimited = shared.allocation.is_unlimited_provider(); + + if shared.connections.is_empty() { + connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); + Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); + if !shared_is_unlimited { + if let Some(name) = shared.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&shared.allocation_id); } + Self::remove_priority_entry( + &mut connections, + &name, + &(shared.priority, Reverse(shared.created_at), shared.allocation_id), + shared.kind, + ); } - Some(allocations) - } else { - None } - }; - - if let Some(allocations) = single_allocations { - for (_, info) in allocations { - debug_if_enabled!( - "Released provider connection {:?} for {}", - info.allocation.get_provider_name().unwrap_or_default(), - sanitize_sensitive_info(&addr.to_string()) - ); - info.allocation.release().await; - } - return; + return Some(shared.allocation); } - // Shared connection - let shared_allocation = { - let mut connections = self.connections.write().await; - - let key = match connections.shared.key_by_addr.get(addr) { - Some(k) => k.clone(), - None => return, // no shared connection - }; - - // Clone the SharedAllocation to avoid double mutable borrow - let mut shared = match connections.shared.by_key.get(&key) { - Some(s) => s.clone(), - None => return, - }; - - // Remove this address from the shared connection set - shared.connections.remove(addr); - // Always remove stale key-by-addr entry - connections.shared.key_by_addr.remove(addr); - - let shared_is_unlimited = shared.allocation.is_unlimited_provider(); - - if shared.connections.is_empty() { - // If this was the last user of the shared allocation: - connections.shared.by_key.remove(&key); - connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); - if !shared_is_unlimited { + // Recompute shared priority from remaining subscribers so preemption decisions + // reflect who is actually still watching the shared stream. Unlimited shared + // streams are not in the preemption index, so this rebalance is skipped. + if !shared_is_unlimited { + let old_priority = shared.priority; + let old_kind = shared.kind; + shared.kind = Self::shared_effective_kind(&shared.connections); + if let Some(new_priority) = Self::shared_effective_priority(&shared.connections, shared.kind) { + shared.priority = new_priority; + if (new_priority, shared.kind) != (old_priority, old_kind) { if let Some(name) = shared.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.retain(|(_, i)| *i != shared.allocation_id); - } Self::remove_priority_entry( &mut connections, &name, - &(shared.priority, Reverse(shared.created_at), shared.allocation_id), + &(old_priority, Reverse(shared.created_at), shared.allocation_id), + old_kind, + ); + Self::upsert_priority_entry( + &mut connections, + &name, + (new_priority, Reverse(shared.created_at), shared.allocation_id), + PriorityOwner::Shared(shared.allocation_id), shared.kind, ); } } - Some(shared.allocation) - } else { - // Recompute shared priority from remaining subscribers so preemption decisions - // reflect who is actually still watching the shared stream. Unlimited - // shared streams are not in the preemption index, so this rebalance - // must be skipped for them. - if !shared_is_unlimited { - let old_priority = shared.priority; - let old_kind = shared.kind; - shared.kind = Self::shared_effective_kind(&shared.connections); - if let Some(new_priority) = Self::shared_effective_priority(&shared.connections, shared.kind) { - shared.priority = new_priority; - if (new_priority, shared.kind) != (old_priority, old_kind) { - if let Some(name) = shared.allocation.get_provider_name() { - Self::remove_priority_entry( - &mut connections, - &name, - &(old_priority, Reverse(shared.created_at), shared.allocation_id), - old_kind, - ); - Self::upsert_priority_entry( - &mut connections, - &name, - (new_priority, Reverse(shared.created_at), shared.allocation_id), - PriorityOwner::Shared(shared.allocation_id), - shared.kind, - ); + } + } + connections.shared.by_key.insert(key, shared); + None + } + + pub fn release_shared_connection(&self, subscriber_id: SharedSubscriberId) { + let _transition = self.lock_capacity_transition(); + self.release_shared_connection_inner(subscriber_id); + } + + fn release_shared_connection_inner(&self, subscriber_id: SharedSubscriberId) { + if let Some(allocation) = self.release_shared_subscriber(subscriber_id) { + debug_if_enabled!( + "Released last shared connection for provider {} (subscriber={subscriber_id})", + allocation.get_provider_name().unwrap_or_default() + ); + allocation.release(); + } + } + + pub fn register_body_owner(&self, alloc_id: AllocationId) -> bool { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle == ConnectionLifecycle::Closing || info.lifecycle == ConnectionLifecycle::Closed { + return false; + } + info.has_body_owner = true; + if info.lifecycle == ConnectionLifecycle::Opening { + info.lifecycle = ConnectionLifecycle::Active; + } + true + } else { + false + } + } + + pub fn mark_opening(&self, alloc_id: AllocationId) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { + info.lifecycle = ConnectionLifecycle::Opening; + } + } + } + + pub fn renew_opening_tokens(&self, alloc_id: AllocationId) -> Option<(CancellationToken, CancellationToken, u64)> { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { + info.lifecycle = ConnectionLifecycle::Opening; + info.open_generation = info.open_generation.wrapping_add(1); + info.reaper_spawned = false; + let cancel_token = CancellationToken::new(); + let completion_token = CancellationToken::new(); + info.cancel_token = cancel_token.clone(); + info.completion_token = completion_token.clone(); + return Some((cancel_token, completion_token, info.open_generation)); + } + } + None + } + + pub fn mark_closing(&self, alloc_id: AllocationId) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + let prio = if let Some(info) = connections.single.get_mut(&alloc_id) { + info.lifecycle = ConnectionLifecycle::Closing; + info.cancel_token.cancel(); + info.allocation + .get_provider_name() + .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)) + } else { + None + }; + if let Some((name, key, kind)) = prio { + Self::remove_priority_entry(&mut connections, &name, &key, kind); + } + } + + pub fn release_connection(&self, addr: &SocketAddr) { + let _transition = self.lock_capacity_transition(); + let single_alloc_ids = { + let mut connections = self.write_connections(); + connections.single_by_addr.remove(addr) + }; + + if let Some(alloc_ids) = single_alloc_ids { + let mut to_reap = Vec::new(); + { + let mut connections = self.write_connections(); + for id in alloc_ids { + if let ReleaseAction::Wait(alloc_id, gen, token) = + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, id) + { + to_reap.push((alloc_id, gen, token)); + } + } + } + for (id, gen, completion_token) in to_reap { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { + rt_handle.spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(id, Some(gen)); } } + }); + } + } + } + + // Shared connections carried by this transport socket. Resolve the subscriber + // ids first, then release each one precisely; two external clients behind one + // reverse proxy hold distinct subscriber ids and never overwrite each other. + for subscriber_id in self.release_shared_by_addr(addr) { + self.release_shared_connection_inner(subscriber_id); + } + } + + fn release_shared_handle_locked( + connections: &mut Connections, + handle: &ProviderHandle, + ) -> Option { + let mut released = None; + let mut released_priority_key: Option<(Arc, PriorityKey, ConnectionKind)> = None; + if let Some(key) = connections.shared.shared_by_allocation_id.remove(&handle.allocation_id) { + if let Some(shared) = connections.shared.by_key.remove(&key) { + Self::unindex_owner(connections, handle.allocation_id, shared.session_owner.as_ref()); + let pkey = (shared.priority, Reverse(shared.created_at), handle.allocation_id); + let shared_kind = shared.kind; + let shared_is_unlimited = shared.allocation.is_unlimited_provider(); + released = Some(shared.allocation); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); + } + if !shared_is_unlimited { + if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&handle.allocation_id); + } + released_priority_key = Some((name, pkey, shared_kind)); + } + } + } + } + if let Some((name, pkey, kind)) = &released_priority_key { + Self::remove_priority_entry(connections, name, pkey, *kind); + } + if let Some(allocation) = released.as_ref() { + allocation.release(); + } + released + } + + pub fn release_handle(&self, handle: &ProviderHandle) { + let _transition = self.lock_capacity_transition(); + let wait_action = { + let mut connections = self.write_connections(); + if connections.single.contains_key(&handle.allocation_id) { + if handle.completion_token.as_ref().is_some_and(CancellationToken::is_cancelled) { + if let Some(info) = connections.single.get_mut(&handle.allocation_id) { + if info.open_generation == handle.open_generation { + info.completion_token.cancel(); + } } } - connections.shared.by_key.insert(key, shared); - None + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, handle.allocation_id) + } else { + Self::release_shared_handle_locked(&mut connections, handle); + ReleaseAction::None } }; - // release allocation - if let Some(allocation) = shared_allocation { - allocation.release().await; - debug_if_enabled!( - "Released last shared connection for provider {}, releasing allocation {}", - allocation.get_provider_name().unwrap_or_default(), - sanitize_sensitive_info(&addr.to_string()) - ); + if let ReleaseAction::Wait(alloc_id, gen, completion_token) = wait_action { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { + rt_handle.spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); + } + } + }); + } } } - pub async fn release_handle(&self, handle: &ProviderHandle) { - let mut released = None; - let mut released_priority_key: Option<(Arc, PriorityKey, ConnectionKind)> = None; - { - let mut connections = self.connections.write().await; - - // Try removing from Single - if let Some(per_addr) = connections.single.get_mut(&handle.client_id) { - if let Some(info) = per_addr.remove(&handle.allocation_id) { - let pkey = (info.priority, Reverse(info.created_at), handle.allocation_id); - released = Some(info.allocation); - let released_kind = info.kind; - if per_addr.is_empty() { - connections.single.remove(&handle.client_id); - } - - // Remove from by_provider index. Skip for unlimited providers: - // they were never inserted by `register_allocation`, so the - // keys/values are not present and a removal would be a no-op - // at best and could race with concurrent unrelated operations - // on the same HashSet entry at worst. - if !released.as_ref().is_some_and(ProviderAllocation::is_unlimited_provider) { - if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&(handle.client_id, handle.allocation_id)); - } - released_priority_key = Some((name, pkey, released_kind)); - } - } - } - } - - if released.is_none() { - // Try removing from Shared - if let Some(key) = connections.shared.shared_by_allocation_id.remove(&handle.allocation_id) { - if let Some(shared) = connections.shared.by_key.remove(&key) { - let pkey = (shared.priority, Reverse(shared.created_at), handle.allocation_id); - released = Some(shared.allocation.clone()); - let shared_is_unlimited = shared.allocation.is_unlimited_provider(); - for addr in shared.connections.keys() { - connections.shared.key_by_addr.remove(addr); - } - if !shared_is_unlimited { - if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.retain(|(_, i)| *i != handle.allocation_id); - } - released_priority_key = Some((name, pkey, shared.kind)); + /// Stops only stale requests of this playback, even on a multiplexed proxy socket. + pub fn release_playback_connections(&self, owner: &str, addrs: &[SocketAddr]) { + let handles = { + let connections = self.read_connections(); + let mut handles = Vec::new(); + for addr in addrs { + if let Some(alloc_ids) = connections.single_by_addr.get(addr) { + for id in alloc_ids { + if let Some(info) = connections.single.get(id) { + if info.session_owner.as_deref() == Some(owner) { + handles.push(ProviderHandle { + playback_request_id: info.playback_request_id, + binding_tag: None, + client_id: info.client_addr, + allocation_id: info.allocation_id, + allocation: info.allocation.clone(), + cancel_token: Some(info.cancel_token.clone()), + completion_token: Some(info.completion_token.clone()), + close_reason: Arc::clone(&info.close_reason), + open_generation: info.open_generation, + }); } } } } } - - // Remove from priority_index - if let Some((name, pkey, kind)) = &released_priority_key { - Self::remove_priority_entry(&mut connections, name, pkey, *kind); + handles + }; + for handle in handles { + handle.set_close_reason(ProviderCloseReason::Superseded); + if let Some(token) = &handle.cancel_token { + token.cancel(); } - } - - if let Some(allocation) = released { - allocation.release().await; + self.release_handle(&handle); } } - pub async fn reclassify_connection(&self, addr: &SocketAddr, kind: ConnectionKind, priority: i8) -> bool { - let mut connections = self.connections.write().await; + /// Stops stale requests of this playback and awaits upstream body closure before releasing capacity. + pub async fn release_playback_connections_await(&self, owner: &str, addrs: &[SocketAddr]) { + let targets = { + let mut connections = self.write_connections(); + let mut targets = Vec::new(); + for addr in addrs { + if let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() { + for id in alloc_ids { + if let Some(info) = connections.single.get_mut(&id) { + if info.session_owner.as_deref() == Some(owner) + && info.lifecycle != ConnectionLifecycle::Closed + { + info.lifecycle = ConnectionLifecycle::Closing; + info.close_reason.store(ProviderCloseReason::Superseded as u8, Ordering::Release); + targets.push(( + info.allocation_id, + info.cancel_token.clone(), + info.completion_token.clone(), + info.priority, + info.created_at, + info.kind, + info.allocation.get_provider_name(), + info.open_generation, + )); + } + } + } + } + } + for (alloc_id, _, _, priority, created_at, kind, provider_name, _) in &targets { + if let Some(name) = provider_name { + Self::remove_priority_entry( + &mut connections, + name, + &(*priority, Reverse(*created_at), *alloc_id), + *kind, + ); + } + } + targets + }; - if let Some(per_addr) = connections.single.get_mut(addr) { - let mut index_updates = Vec::new(); - for (allocation_id, info) in per_addr.iter_mut() { + if targets.is_empty() { + return; + } + + for (_, cancel_token, _, _, _, _, _, _) in &targets { + cancel_token.cancel(); + } + + let futures: Vec<_> = targets.iter().map(|(_, _, token, _, _, _, _, _)| token.cancelled()).collect(); + let _ = tokio::time::timeout(Duration::from_secs(1), futures::future::join_all(futures)).await; + + for (alloc_id, _, completion_token, _, _, _, _, gen) in targets { + if completion_token.is_cancelled() { + self.core.complete_release_with_generation(alloc_id, Some(gen)); + } else { + log::warn!("Superseded provider connection {alloc_id} did not close within deadline, retaining slot as Closing until reaped"); + // Atomically claim the reaper role under the lock before spawning, so that + // if this future was dropped between collection and here, release_handle can + // still install its own reaper (reaper_spawned remains false until now). + let should_spawn = { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + connections.single.get_mut(&alloc_id).is_some_and(|info| { + if info.reaper_spawned { + false + } else { + info.reaper_spawned = true; + true + } + }) + }; + if should_spawn { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); + } + } + }); + } + } + } + } + + pub fn reclassify_connection(&self, addr: &SocketAddr, kind: ConnectionKind, priority: i8) -> bool { + self.reclassify_connection_for_owner(addr, None, kind, priority) + } + + pub fn reclassify_connection_for_owner( + &self, + addr: &SocketAddr, + owner: Option<&str>, + kind: ConnectionKind, + priority: i8, + ) -> bool { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + + let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() else { + return false; + }; + let mut index_updates = Vec::new(); + for allocation_id in alloc_ids { + if let Some(info) = connections.single.get_mut(&allocation_id) { + if info.session_owner.as_deref() != owner { + continue; + } if let Some(provider_name) = info.allocation.get_provider_name() { - let old_key = (info.priority, Reverse(info.created_at), *allocation_id); - let owner = PriorityOwner::Single(*addr); + let old_key = (info.priority, Reverse(info.created_at), allocation_id); + let p_owner = PriorityOwner::Single(allocation_id); let old_kind = info.kind; info.kind = kind; info.priority = priority; - let new_key = (info.priority, Reverse(info.created_at), *allocation_id); + let new_key = (info.priority, Reverse(info.created_at), allocation_id); // Skip preemption-index updates for unlimited providers: they // are intentionally absent from `by_provider` / `priority_index` // / `soft_priority_index`, so the old entry was never inserted. // Performing the upsert here would re-introduce them into the // preemption index and contradict `max_connections: 0`. if !info.allocation.is_unlimited_provider() { - index_updates.push((provider_name, old_key, old_kind, new_key, owner, info.kind)); + index_updates.push((provider_name, old_key, old_kind, new_key, p_owner, info.kind)); } } } - for (provider_name, old_key, old_kind, new_key, owner, new_kind) in index_updates { - Self::remove_priority_entry(&mut connections, &provider_name, &old_key, old_kind); - Self::upsert_priority_entry(&mut connections, &provider_name, new_key, owner, new_kind); - } - return true; } + for (provider_name, old_key, old_kind, new_key, p_owner, new_kind) in index_updates { + Self::remove_priority_entry(&mut connections, &provider_name, &old_key, old_kind); + Self::upsert_priority_entry(&mut connections, &provider_name, new_key, p_owner, new_kind); + } + true + } - let Some(shared_key) = connections.shared.key_by_addr.get(addr).cloned() else { + pub fn reclassify_shared_connection( + &self, + subscriber_id: SharedSubscriberId, + kind: ConnectionKind, + priority: i8, + ) -> bool { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + let shared_key = connections.shared.key_by_subscriber.get(&subscriber_id).cloned(); + let Some(shared_key) = shared_key else { return false; }; let Some(shared_allocation) = connections.shared.by_key.get_mut(&shared_key) else { return false; }; - let Some(subscriber) = shared_allocation.connections.get_mut(addr) else { - return false; - }; let shared_is_unlimited = shared_allocation.allocation.is_unlimited_provider(); let old_priority = shared_allocation.priority; let old_kind = shared_allocation.kind; + let Some(subscriber) = shared_allocation.connections.get_mut(&subscriber_id) else { + return false; + }; subscriber.kind = kind; subscriber.priority = priority; shared_allocation.kind = Self::shared_effective_kind(&shared_allocation.connections); @@ -1345,134 +2464,90 @@ impl ActiveProviderManager { true } - #[allow(clippy::too_many_lines)] - pub async fn make_shared_connection(&self, addr: &SocketAddr, key: &str) { - let extras = { - let mut connections = self.connections.write().await; - let mut extras = Vec::new(); - let shared_key: Arc = Arc::from(key); - - // Find the allocation to promote (must be single) - // Logic change: we must find the specific allocation if multiple exist, but usually per client only 1 active? - // Existing logic assumes one. - let handle = if let Some(m) = connections.single.get_mut(addr) { - if m.is_empty() { - None - } else { - let mut iter = m.drain(); - if let Some((id, info)) = iter.next() { - // Collect others as extras to release - let extra_entries: Vec<_> = iter.collect(); - - // Cleanup indices. Skip for unlimited providers: their - // entries were never inserted into `by_provider` / - // `priority_index` / `soft_priority_index`, so the - // removal would be a no-op. - if !info.allocation.is_unlimited_provider() { - if let Some(name) = info.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&(*addr, id)); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(info.priority, Reverse(info.created_at), id), - info.kind, - ); - } - } - // Remove extras from provider-specific indexes. - for (extra_id, extra_info) in &extra_entries { - if !extra_info.allocation.is_unlimited_provider() { - if let Some(extra_provider_name) = extra_info.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&extra_provider_name) { - list.remove(&(*addr, *extra_id)); - } - Self::remove_priority_entry( - &mut connections, - &extra_provider_name, - &(extra_info.priority, Reverse(extra_info.created_at), *extra_id), - extra_info.kind, - ); - } - } - } - - for (_, extra_info) in extra_entries { - extra_info.cancel_token.cancel(); - extras.push(extra_info.allocation); - } - - connections.single.remove(addr); // Map is drained/empty now - - Some(( - ProviderHandle::new(*addr, id, info.allocation, Some(info.cancel_token)), - info.priority, - info.kind, - info.created_at, - info.session_owner, - )) - } else { - None - } - } - } else { - None - }; - - if let Some(handle) = &handle { - let provider_name = handle.0.allocation.get_provider_name().unwrap_or_default(); - debug_if_enabled!( - "Shared connection: promoted addr {addr} provider={} key={}", - sanitize_sensitive_info(&provider_name), - sanitize_sensitive_info(key) - ); - - connections.shared.by_key.insert( - Arc::clone(&shared_key), - SharedAllocation { - allocation_id: handle.0.allocation_id, - allocation: handle.0.allocation.clone(), - connections: HashMap::from([(*addr, SharedSubscriber { priority: handle.1, kind: handle.2 })]), - priority: handle.1, - kind: handle.2, - created_at: handle.3, - cancel_token: handle.0.cancel_token.clone(), - session_owner: handle.4.clone(), - }, - ); - connections.shared.key_by_addr.insert(*addr, Arc::clone(&shared_key)); - connections.shared.shared_by_allocation_id.insert(handle.0.allocation_id, shared_key); - - // Insert new shared entry into priority_index. Skip for - // unlimited providers: they are not subject to preemption, so - // they must not appear in the priority index. - if !handle.0.allocation.is_unlimited_provider() { - Self::upsert_priority_entry( - &mut connections, - &provider_name, - (handle.1, Reverse(handle.3), handle.0.allocation_id), - PriorityOwner::Shared(handle.0.allocation_id), - handle.2, - ); - } - } - extras - }; - - for allocation in extras { - allocation.release().await; + /// Promotes exactly the allocation owned by this handle into a shared origin. + pub fn make_shared_connection( + &self, + handle: &ProviderHandle, + key: &str, + subscriber_id: SharedSubscriberId, + ) -> bool { + if self.is_shutting_down.load(Ordering::Acquire) { + return false; } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return false; + } + let mut connections = self.write_connections(); + if connections.shared.by_key.contains_key(key) { + return false; + } + let Some(info) = connections.single.remove(&handle.allocation_id) else { + return false; + }; + if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { + set.remove(&handle.allocation_id); + if set.is_empty() { + connections.single_by_addr.remove(&info.client_addr); + } + } + let provider_name = info.allocation.get_provider_name().unwrap_or_default(); + if !info.allocation.is_unlimited_provider() { + if let Some(list) = connections.by_provider.get_mut(&provider_name) { + list.remove(&handle.allocation_id); + } + Self::remove_priority_entry( + &mut connections, + &provider_name, + &(info.priority, Reverse(info.created_at), handle.allocation_id), + info.kind, + ); + Self::upsert_priority_entry( + &mut connections, + &provider_name, + (info.priority, Reverse(info.created_at), handle.allocation_id), + PriorityOwner::Shared(handle.allocation_id), + info.kind, + ); + } + let shared_key: Arc = Arc::from(key); + connections.shared.by_key.insert( + Arc::clone(&shared_key), + SharedAllocation { + allocation_id: handle.allocation_id, + allocation: info.allocation, + connections: HashMap::from([( + subscriber_id, + SharedSubscriber { priority: info.priority, kind: info.kind, addr: handle.client_id }, + )]), + priority: info.priority, + kind: info.kind, + created_at: info.created_at, + cancel_token: Some(info.cancel_token), + session_owner: info.session_owner, + }, + ); + connections.shared.key_by_subscriber.insert(subscriber_id, Arc::clone(&shared_key)); + connections.shared.shared_by_allocation_id.insert(handle.allocation_id, shared_key); + true } - pub async fn add_shared_connection( + pub fn add_shared_connection( &self, addr: &SocketAddr, + subscriber_id: SharedSubscriberId, key: &str, priority: i8, kind: ConnectionKind, ) -> Result<(), String> { - let mut connections = self.connections.write().await; + if self.is_shutting_down.load(Ordering::Acquire) { + return Err("Provider manager is shutting down".to_string()); + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return Err("Provider manager is shutting down".to_string()); + } + let mut connections = self.write_connections(); // Extract metadata before taking a second mutable borrow on `connections`. let metadata = connections.shared.by_key.get(key).map(|s| { @@ -1508,7 +2583,7 @@ impl ActiveProviderManager { return Err(err); }; - shared_allocation.connections.insert(*addr, SharedSubscriber { priority, kind }); + shared_allocation.connections.insert(subscriber_id, SharedSubscriber { priority, kind, addr: *addr }); let new_kind = Self::shared_effective_kind(&shared_allocation.connections); let new_priority = Self::shared_effective_priority(&shared_allocation.connections, new_kind); shared_allocation.kind = new_kind; @@ -1537,16 +2612,16 @@ impl ActiveProviderManager { ); } - connections.shared.key_by_addr.insert(*addr, Arc::from(key)); + connections.shared.key_by_subscriber.insert(subscriber_id, Arc::from(key)); Ok(()) } - pub async fn get_provider_connections_count(&self) -> usize { self.providers.active_connection_count().await } + pub fn get_provider_connections_count(&self) -> usize { self.providers.active_connection_count() } - pub async fn provider_capacities_for_input(&self, input_name: &Arc) -> Vec<(Arc, usize, usize)> { + pub fn provider_capacities_for_input(&self, input_name: &Arc) -> Vec<(Arc, usize, usize)> { let mut result = Vec::new(); for provider_name in self.providers.provider_names_for_input(input_name) { - if let Some((current, max)) = self.providers.provider_capacity(&provider_name).await { + if let Some((current, max)) = self.providers.provider_capacity(&provider_name) { result.push((provider_name, current, max)); } } @@ -1556,8 +2631,8 @@ impl ActiveProviderManager { #[cfg(test)] mod tests { - use super::{ActiveProviderManager, ConnectionKind}; - use crate::EventManager; + use super::{ActiveProviderManager, ConnectionKind, PlaybackLeaseRef}; + use crate::{EventManager, SharedStreamManager}; use arc_swap::{ArcSwap, ArcSwapOption}; use shared::{ defaults::{default_probe_user_priority, default_user_priority}, @@ -1567,12 +2642,13 @@ mod tests { use std::{ collections::{HashMap, HashSet}, net::SocketAddr, - sync::Arc, + sync::{Arc, Weak}, time::Duration, }; use tuliprox_core::{ model::{ - AppConfig, Config, ConfigInput, ConfigInputAlias, MediaToolCapabilities, ProviderAllocation, SourcesConfig, + AppConfig, Config, ConfigInput, ConfigInputAlias, MediaToolCapabilities, PlaybackKind, + PlaybackRequestOutcome, ProviderAllocation, SharedSubscriberId, SourcesConfig, }, utils::FileLockManager, }; @@ -1662,6 +2738,318 @@ mod tests { fn create_test_app_config_single_unlimited_provider_pool() -> AppConfig { build_test_app_config(None, 0) } + /// Pool where the higher-priority provider (A) and its lower-priority alias (B) + /// each carry their own capacity, mirroring the reported A=2 / B=3 setup. + fn create_test_app_config_with_pool(primary_max: u16, alias_max: u16) -> AppConfig { + build_test_app_config( + Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: "http://provider-2.example".to_string(), + username: Some("user2".to_string()), + password: Some("pass2".to_string()), + priority: 1, + max_connections: alias_max, + exp_date: None, + enabled: true, + stalker: None, + }]), + primary_max, + ) + } + + #[test] + fn shared_stream_manager_backref_is_weak_and_does_not_leak() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared = Arc::new(SharedStreamManager::new(Arc::clone(&provider))); + provider.set_shared_stream_manager(&shared); + + // The only strong reference to the shared manager is the local `shared`; the + // provider holds a Weak backref, so dropping `shared` releases both managers. + drop(shared); + assert!(provider.shared_stream_manager.get().and_then(Weak::upgrade).is_none()); + } + + #[tokio::test] + async fn confirmed_playbacks_fill_higher_priority_provider_before_alias() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let mut selected = Vec::new(); + for index in 0..5 { + let owner = format!("playback-{index}"); + let addr = SocketAddr::from(([172, 18, 0, 9], 50_000 + index)); + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(&owner), + ) + .expect("pool has capacity for five confirmed playbacks"); + let provider = handle.allocation.get_provider_name().expect("provider name"); + // Only real media activity confirms a playback and lets it reserve capacity. + manager.confirm_playback_activity(&owner); + selected.push(provider.to_string()); + } + + assert_eq!(selected, ["provider_1", "provider_1", "provider_2", "provider_2", "provider_2"]); + } + + #[tokio::test] + async fn concurrent_proxy_requests_fill_exact_pool_capacity() -> Result<(), Box> { + let manager = Arc::new(ActiveProviderManager::new( + &create_test_app_config_with_pool(2, 3), + &Arc::new(EventManager::new()), + )); + let addr = SocketAddr::from(([127, 0, 0, 1], 50001)); + let mut tasks = tokio::task::JoinSet::new(); + for attempt in 0..32 { + let manager = Arc::clone(&manager); + tasks.spawn(async move { + manager.acquire_connection_with_grace_for_session( + &Arc::from("provider_1"), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(&format!("playback-{attempt}")), + ) + }); + } + let mut handles = Vec::new(); + while let Some(result) = tasks.join_next().await { + if let Some(handle) = result? { + handles.push(handle); + } + } + assert_eq!(handles.len(), 5); + assert_eq!( + handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_1")).count(), + 2 + ); + assert_eq!( + handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_2")).count(), + 3 + ); + for handle in handles { + manager.release_handle(&handle); + } + assert_eq!(manager.get_provider_connections_count(), 0); + Ok(()) + } + + #[tokio::test] + async fn forced_reopen_and_late_cleanup_preserve_other_proxy_requests() -> Result<(), Box> { + let manager = + ActiveProviderManager::new(&create_test_app_config_with_pool(2, 3), &Arc::new(EventManager::new())); + let addr = SocketAddr::from(([127, 0, 0, 1], 50002)); + let input = Arc::from("provider_1"); + let first = manager + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first")) + .ok_or("first allocation missing")?; + let other = manager + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("other")) + .ok_or("other allocation missing")?; + manager.release_playback_connections("first", &[addr]); + assert!(first.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + assert!(!other.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + assert_eq!(manager.get_provider_connections_count(), 1); + let replacement = manager + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first")) + .ok_or("replacement missing")?; + manager.refresh_adaptive_playback_lease(&input, "first", tuliprox_core::model::PlaybackKind::LiveHls, 15); + manager.confirm_playback_activity("first"); + manager.finish_playback_request("first", PlaybackRequestOutcome::ProviderFailed, None); + assert_eq!(manager.provider_lease_usage(&input).active, 1); + manager.release_handle(&replacement); + manager.release_handle(&other); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn manifest_retries_without_media_do_not_reserve_capacity() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "hls-cache:shared-session"; + let addr = SocketAddr::from(([172, 18, 0, 9], 51_000)); + + // Five manifest responses without a single media segment, all on one owner. + for _ in 0..5 { + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("manifest start allocates on the preferred provider"); + assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1")); + manager.refresh_provider_reservation(&input_name, owner, 15); + manager.release_connection(&addr); + } + + // Unconfirmed leases never reserve capacity, so an unrelated client is served + // by the higher-priority provider even though its own counters read zero. + let other = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 51_001)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("other-client"), + ) + .expect("unrelated client must still get the preferred provider"); + assert_eq!(other.allocation.get_provider_name().as_deref(), Some("provider_1")); + + // The abandoned starts stop holding anything once their startup deadline passes. + tokio::time::advance(Duration::from_secs(6)).await; + manager.prune_expired_leases_now(); + assert!(!manager.is_provider_reserved_for_other_session(&input_name, Some("other-client"))); + } + + #[tokio::test(start_paused = true)] + async fn identified_reservation_recreates_a_cleared_lease() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "hls-cache:restore-session"; + let addr = SocketAddr::from(([172, 18, 0, 9], 53_000)); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("allocate"); + manager.refresh_identified_provider_reservation( + &input_name, + owner, + tuliprox_core::model::PlaybackKind::Catchup, + 15, + ); + assert_eq!(manager.provider_lease_usage(&input_name).starting, 1); + + // Clearing removes the lease entirely; recreating must bring it back rather + // than silently no-op like a plain adaptive refresh would. + let binding_tag = manager.binding_tag_for_owner(owner); + manager.clear_identified_provider_reservation(owner, &input_name, binding_tag); + assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); + + manager.refresh_identified_provider_reservation( + &input_name, + owner, + tuliprox_core::model::PlaybackKind::Catchup, + 15, + ); + assert_eq!(manager.provider_lease_usage(&input_name).starting, 1); + + manager.release_handle(&handle); + } + + #[tokio::test(start_paused = true)] + async fn reservations_do_not_survive_after_counters_reach_zero() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "finished-playback"; + let addr = SocketAddr::from(([172, 18, 0, 9], 52_000)); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("first playback acquires the preferred provider"); + assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1")); + manager.refresh_provider_reservation(&input_name, owner, 15); + manager.confirm_playback_activity(owner); + manager.release_connection(&addr); + + // Live TS is not reconnect capable: the confirmed lease is dropped on release, + // so the provider is immediately free again for the next higher-priority client. + manager.finish_playback_request(owner, PlaybackRequestOutcome::Completed, None); + let next = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 52_001)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("next-client"), + ) + .expect("freed capacity must be selectable again"); + assert_eq!(next.allocation.get_provider_name().as_deref(), Some("provider_1")); + } + + #[tokio::test] + async fn two_playbacks_sharing_one_proxy_socket_keep_separate_slots() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + // Both external clients arrive through the same reverse-proxy peer socket. + let proxy_addr = SocketAddr::from(([172, 18, 0, 9], 53_000)); + + let first = manager + .acquire_connection_with_grace_for_session( + &input_name, + &proxy_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some("device-one"), + ) + .expect("first device behind the proxy acquires a slot"); + let second = manager + .acquire_connection_with_grace_for_session( + &input_name, + &proxy_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some("device-two"), + ) + .expect("second device behind the proxy acquires its own slot"); + assert_ne!(first.allocation_id, second.allocation_id); + manager.confirm_playback_activity("device-one"); + manager.confirm_playback_activity("device-two"); + + // Releasing one playback must free exactly its own allocation and leave the + // other device's slot on the same transport untouched. + manager.release_handle(&first); + manager.finish_playback_request("device-one", PlaybackRequestOutcome::Completed, None); + let remaining = manager.provider_capacities_for_input(&input_name); + let primary = remaining.iter().find(|(name, _, _)| name.as_ref() == "provider_1").expect("primary pool entry"); + assert_eq!(primary.1, 1, "exactly one slot must remain in use on the shared socket"); + assert!(manager.read_leases().lease_of_owner("device-one").is_none()); + assert!(manager.read_leases().lease_of_owner("device-two").is_some()); + + manager.release_handle(&second); + manager.finish_playback_request("device-two", PlaybackRequestOutcome::Completed, None); + } + #[tokio::test] async fn unlimited_provider_connection_is_not_in_priority_index() { let app_cfg = create_test_app_config_single_unlimited_provider_pool(); @@ -1675,19 +3063,16 @@ mod tests { manager .acquire_connection(&input_name, &addr_1, default_user_priority(), ConnectionKind::Normal) - .await .expect("acquire #1 on unlimited provider"); manager .acquire_connection(&input_name, &addr_2, default_user_priority(), ConnectionKind::Normal) - .await .expect("acquire #2 on unlimited provider"); manager .acquire_connection(&input_name, &addr_3, default_user_priority(), ConnectionKind::Normal) - .await .expect("acquire #3 on unlimited provider"); { - let connections = manager.connections.read().await; + let connections = manager.read_connections(); let priority_tree = connections.priority_index.get(&input_name); assert!( priority_tree.is_none_or(std::collections::BTreeMap::is_empty), @@ -1705,9 +3090,9 @@ mod tests { ); } - manager.release_connection(&addr_1).await; - manager.release_connection(&addr_2).await; - manager.release_connection(&addr_3).await; + manager.release_connection(&addr_1); + manager.release_connection(&addr_2); + manager.release_connection(&addr_3); } #[tokio::test] @@ -1723,19 +3108,17 @@ mod tests { // Acquire one low-priority and one high-priority connection on the unlimited provider. let low = manager .acquire_connection(&input_name, &low_addr, 50, ConnectionKind::Normal) - .await .expect("low-priority acquire on unlimited provider"); let low_token = low.cancel_token.clone().expect("cancel token for low-priority connection"); let _high = manager .acquire_connection(&input_name, &high_addr, 0, ConnectionKind::Normal) - .await .expect("high-priority acquire on unlimited provider"); // A request at an even higher priority must not be able to preempt the unlimited // provider's connection, because the connection is intentionally absent from the // preemption indices. let candidate = { - let connections = manager.connections.read().await; + let connections = manager.read_connections(); manager.select_preemption_candidate(&connections, &input_name, 0, ConnectionKind::Normal, &HashSet::new()) }; assert!( @@ -1749,8 +3132,8 @@ mod tests { "low-priority unlimited-provider connection must not be cancelled by preemption" ); - manager.release_connection(&low_addr).await; - manager.release_connection(&high_addr).await; + manager.release_connection(&low_addr); + manager.release_connection(&high_addr); } #[tokio::test] @@ -1765,24 +3148,21 @@ mod tests { let first_alloc = manager .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("client1 initial allocation"); let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected"); assert_eq!(pinned_provider.as_ref(), "provider_1"); // provider_1 has max_connections=1 and is already in use by client1 - let forced = manager - .force_exact_acquire_connection( - &pinned_provider, - &client_2_addr, - default_user_priority(), - ConnectionKind::Normal, - ) - .await; + let forced = manager.force_exact_acquire_connection( + &pinned_provider, + &client_2_addr, + default_user_priority(), + ConnectionKind::Normal, + ); assert!(forced.is_none(), "forced exact acquire must not over-allocate busy provider"); - manager.release_connection(&client_1_addr).await; - manager.release_connection(&client_2_addr).await; + manager.release_connection(&client_1_addr); + manager.release_connection(&client_2_addr); } #[tokio::test] @@ -1798,35 +3178,32 @@ mod tests { // Step 1: Client1 starts movie -> provider_1 let first_alloc = manager .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("client1 initial allocation"); assert_eq!(first_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); // Step 2: Client1 stops -> release provider_1 - manager.release_connection(&client_1_addr).await; + manager.release_connection(&client_1_addr); // Step 3: Client2 starts live -> provider_1 let live_alloc = manager .acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("client2 live allocation"); let busy_provider = live_alloc.allocation.get_provider_name().expect("provider name expected"); assert_eq!(busy_provider.as_ref(), input_name.as_ref()); - assert!(manager.is_exhausted(&busy_provider).await); + assert!(manager.is_exhausted(&busy_provider)); // Step 4: Client1 restarts same movie. // This emulates force-session fallback path by acquiring without provider grace. let fallback_alloc = manager .acquire_connection_with_grace(&input_name, &client_1_addr, false, 0, ConnectionKind::Normal) - .await .expect("client1 fallback allocation without grace"); let fallback_provider = fallback_alloc.allocation.get_provider_name().expect("fallback provider expected"); assert_ne!(fallback_provider.as_ref(), busy_provider.as_ref()); assert_eq!(fallback_provider.as_ref(), "provider_2"); - manager.release_connection(&client_1_addr).await; - manager.release_connection(&client_2_addr).await; + manager.release_connection(&client_1_addr); + manager.release_connection(&client_2_addr); } #[tokio::test] @@ -1842,7 +3219,6 @@ mod tests { // Initial playback for client1. let first_alloc = manager .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("client1 initial allocation"); let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected"); assert_eq!(pinned_provider.as_ref(), "provider_1"); @@ -1850,7 +3226,6 @@ mod tests { // Another client occupies the alternate account while client1 keeps seeking. let second_alloc = manager .acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("client2 allocation"); let second_provider = second_alloc.allocation.get_provider_name().expect("provider name expected"); assert_eq!(second_provider.as_ref(), "provider_2"); @@ -1858,7 +3233,7 @@ mod tests { // Simulate repeated seek/range reconnects for client1: // release old connection for the same client, then force exact pinned provider. for _ in 0..3 { - manager.release_connection(&client_1_addr).await; + manager.release_connection(&client_1_addr); let seek_alloc = manager .force_exact_acquire_connection( &pinned_provider, @@ -1866,15 +3241,14 @@ mod tests { default_user_priority(), ConnectionKind::Normal, ) - .await .expect("seek reacquire should stay on pinned provider"); let seek_provider = seek_alloc.allocation.get_provider_name().expect("provider name expected"); assert_eq!(seek_provider.as_ref(), pinned_provider.as_ref()); } // Stream stop / cleanup. - manager.release_connection(&client_1_addr).await; - manager.release_connection(&client_2_addr).await; + manager.release_connection(&client_1_addr); + manager.release_connection(&client_2_addr); } #[tokio::test] @@ -1888,7 +3262,6 @@ mod tests { let probe_handle = manager .acquire_connection_for_probe(&input_name, default_probe_user_priority()) - .await .expect("probe allocation should succeed"); let probe_token = probe_handle.cancel_token.clone().expect("probe handle must carry cancel token"); @@ -1901,7 +3274,6 @@ mod tests { default_user_priority(), ConnectionKind::Normal, ) - .await .expect("user allocation should preempt probe"); assert_eq!(user_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); @@ -1909,7 +3281,7 @@ mod tests { tokio::task::yield_now().await; assert!(probe_token.is_cancelled(), "probe token should be cancelled immediately after preemption"); - manager.release_connection(&user_addr).await; + manager.release_connection(&user_addr); } #[tokio::test(start_paused = true)] @@ -1924,7 +3296,8 @@ mod tests { let addr_1: SocketAddr = "127.0.0.1:43101".parse().unwrap(); let addr_2: SocketAddr = "127.0.0.1:43102".parse().unwrap(); - manager.refresh_provider_reservation(&input_name, owner_1, 15).await; + manager.refresh_provider_reservation(&input_name, owner_1, 15); + manager.confirm_playback_activity(owner_1); let first = manager .acquire_connection_with_grace_for_session( @@ -1935,21 +3308,18 @@ mod tests { ConnectionKind::Normal, Some(owner_1), ) - .await .expect("reserved owner should reacquire its provider"); assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.release_connection(&addr_1).await; + manager.release_connection(&addr_1); - let blocked = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ) - .await; + let blocked = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ); assert!(blocked.is_none(), "other sessions must not take a reserved provider before TTL expiry"); tokio::time::advance(Duration::from_secs(16)).await; @@ -1963,10 +3333,102 @@ mod tests { ConnectionKind::Normal, Some(owner_2), ) - .await .expect("reservation should expire after TTL"); assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.release_connection(&addr_2).await; + manager.release_connection(&addr_2); + } + + #[tokio::test(start_paused = true)] + async fn denied_confirmation_followed_by_refresh_cannot_reserve_foreign_slot() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner_x = "session-owner-x"; + let owner_y = "session-owner-y"; + let owner_z = "session-owner-z"; + let addr_y: SocketAddr = "127.0.0.1:43310".parse().unwrap(); + let addr_z: SocketAddr = "127.0.0.1:43311".parse().unwrap(); + + // Y holds the only provider slot. + let y = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_y, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_y), + ) + .expect("y acquires the single slot"); + + // X starts a lease and confirms media while Y holds the slot, so its + // reservation right is denied. A later refresh must not restore it. + manager.refresh_provider_reservation(&input_name, owner_x, 15); + manager.confirm_playback_activity(owner_x); + manager.refresh_provider_reservation(&input_name, owner_x, 15); + + manager.release_handle(&y); + + // If X's refresh had re-granted the denied reservation, Z would be blocked here. + let z = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_z, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_z), + ); + assert!(z.is_some(), "a denied reservation must not be restored by a later refresh"); + if let Some(z) = z { + manager.release_handle(&z); + } + } + + #[tokio::test(start_paused = true)] + async fn untagged_clear_cannot_delete_successor_reservation() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "session-owner"; + let addr: SocketAddr = "127.0.0.1:43320".parse().unwrap(); + + // First incarnation acquires, then is removed entirely. + let first = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("first acquire"); + manager.release_handle(&first); + manager.clear_provider_reservation(owner); + + // Second incarnation acquires a fresh lease on the same account. + let second = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("second acquire"); + + // An untagged clear has no delete right and must not remove the successor. + manager.clear_identified_provider_reservation(owner, &input_name, None); + assert!( + manager.binding_tag_for_owner(owner).is_some(), + "an untagged clear must not delete the successor reservation" + ); + + manager.release_handle(&second); + manager.clear_provider_reservation(owner); } #[tokio::test(start_paused = true)] @@ -1988,10 +3450,10 @@ mod tests { ConnectionKind::Normal, Some(&owner), ) - .await .expect("preferred provider should retain free capacity for another session"); assert_eq!(allocation.allocation.get_provider_name().as_deref(), Some("provider_1")); - manager.refresh_provider_reservation(&input_name, &owner, 15).await; + manager.refresh_provider_reservation(&input_name, &owner, 15); + manager.confirm_playback_activity(&owner); } let fallback_addr = SocketAddr::from(([127, 0, 0, 1], 43203)); @@ -2004,12 +3466,11 @@ mod tests { ConnectionKind::Normal, Some("session-owner-3"), ) - .await .expect("lower-priority provider should be used after preferred capacity is exhausted"); assert_eq!(fallback.allocation.get_provider_name().as_deref(), Some("provider_2")); - manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 43200))).await; - manager.release_connection(&fallback_addr).await; + manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 43200))); + manager.release_connection(&fallback_addr); let foreign_after_release = manager .acquire_connection_with_grace_for_session( @@ -2020,7 +3481,6 @@ mod tests { ConnectionKind::Normal, Some("session-owner-4"), ) - .await .expect("idle reservation should protect one preferred-provider slot"); assert_eq!(foreign_after_release.allocation.get_provider_name().as_deref(), Some("provider_2")); @@ -2034,13 +3494,12 @@ mod tests { ConnectionKind::Normal, Some("session-owner-0"), ) - .await .expect("reservation owner should reclaim its preferred-provider slot"); assert_eq!(reserved_owner.allocation.get_provider_name().as_deref(), Some("provider_1")); for index in 1..5 { let addr = SocketAddr::from(([127, 0, 0, 1], 43200 + index)); - manager.release_connection(&addr).await; + manager.release_connection(&addr); } } @@ -2056,7 +3515,7 @@ mod tests { let addr_1: SocketAddr = "127.0.0.1:43121".parse().unwrap(); let addr_2: SocketAddr = "127.0.0.1:43122".parse().unwrap(); - manager.refresh_provider_reservation(&input_name, owner_1, 15).await; + manager.refresh_provider_reservation(&input_name, owner_1, 15); let first = manager .acquire_connection_with_grace_for_session( @@ -2067,7 +3526,6 @@ mod tests { ConnectionKind::Normal, Some(owner_1), ) - .await .expect("reserved owner should reacquire its unlimited provider"); assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); @@ -2080,12 +3538,11 @@ mod tests { ConnectionKind::Normal, Some(owner_2), ) - .await .expect("other sessions should not be blocked by reservations on unlimited providers"); assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.release_connection(&addr_1).await; - manager.release_connection(&addr_2).await; + manager.release_connection(&addr_1); + manager.release_connection(&addr_2); } #[tokio::test(start_paused = true)] @@ -2101,33 +3558,30 @@ mod tests { let owner_channel_1 = "client|ua|user|100"; let owner_channel_2 = "client|ua|user|200"; - manager.refresh_provider_reservation(&input_name, owner_channel_1, 15).await; + manager.refresh_provider_reservation(&input_name, owner_channel_1, 15); + manager.confirm_playback_activity(owner_channel_1); - let blocked = manager - .acquire_connection_with_grace_for_session( - &input_name, - &second_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_2), - ) - .await; + let blocked = manager.acquire_connection_with_grace_for_session( + &input_name, + &second_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_2), + ); assert!(blocked.is_none(), "a related but distinct playback must not steal an idle reservation"); - manager.clear_provider_reservation(owner_channel_1).await; - let acquired = manager - .acquire_connection_with_grace_for_session( - &input_name, - &second_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_2), - ) - .await; + manager.clear_provider_reservation(owner_channel_1); + let acquired = manager.acquire_connection_with_grace_for_session( + &input_name, + &second_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_2), + ); assert!(acquired.is_some(), "explicitly clearing the old playback should release its provider"); - manager.release_connection(&second_addr).await; + manager.release_connection(&second_addr); } #[tokio::test(start_paused = true)] @@ -2147,53 +3601,51 @@ mod tests { let owner_channel_1 = "proxy|player|user|100"; let owner_channel_2 = "proxy|player|user|200"; - let first = manager - .acquire_connection_with_grace_for_session( - &input_name, - &first_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_1), - ) - .await; + let first = manager.acquire_connection_with_grace_for_session( + &input_name, + &first_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_1), + ); assert!(first.is_some(), "first playback should acquire the preferred provider"); let Some(first) = first else { return; }; assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.refresh_provider_reservation(&input_name, owner_channel_1, 15).await; - manager.release_connection(&first_addr).await; + manager.refresh_provider_reservation(&input_name, owner_channel_1, 15); + manager.confirm_playback_activity(owner_channel_1); + manager.release_connection(&first_addr); - let second = manager - .acquire_connection_with_grace_for_session( - &input_name, - &second_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_2), - ) - .await; + let second = manager.acquire_connection_with_grace_for_session( + &input_name, + &second_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_2), + ); assert!(second.is_some(), "parallel playback should acquire the remaining provider"); let Some(second) = second else { return; }; assert_eq!(second.allocation.get_provider_name().as_deref(), Some(provider_2.as_ref())); - manager.refresh_provider_reservation(&provider_2, owner_channel_2, 15).await; + manager.refresh_provider_reservation(&provider_2, owner_channel_2, 15); + manager.confirm_playback_activity(owner_channel_2); - let reservations = manager.reservations.read().await; - assert!(reservations.get(&input_name).is_some_and(|entries| entries.contains_key(owner_channel_1))); - assert!(reservations.get(&provider_2).is_some_and(|entries| entries.contains_key(owner_channel_2))); - drop(reservations); + let leases = manager.read_leases(); + assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref())); + assert_eq!(leases.provider_for_owner(owner_channel_2).as_deref(), Some(provider_2.as_ref())); + drop(leases); - manager.clear_provider_reservation(owner_channel_2).await; - let reservations = manager.reservations.read().await; - assert!(reservations.get(&input_name).is_some_and(|entries| entries.contains_key(owner_channel_1))); - assert!(!reservations.values().any(|entries| entries.contains_key(owner_channel_2))); - drop(reservations); + manager.clear_provider_reservation(owner_channel_2); + let leases = manager.read_leases(); + assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref())); + assert!(leases.lease_of_owner(owner_channel_2).is_none()); + drop(leases); - manager.release_connection(&second_addr).await; + manager.release_connection(&second_addr); } #[tokio::test(start_paused = true)] @@ -2207,32 +3659,29 @@ mod tests { let owner_2 = "session-owner-2"; let addr_2: SocketAddr = "127.0.0.1:43132".parse().unwrap(); - manager.refresh_provider_reservation(&input_name, owner_1, 15).await; + manager.refresh_provider_reservation(&input_name, owner_1, 15); + manager.confirm_playback_activity(owner_1); - let blocked = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ) - .await; + let blocked = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ); assert!(blocked.is_none(), "reservation should initially block another session"); - manager.clear_provider_reservation(owner_1).await; + manager.clear_provider_reservation(owner_1); - let acquired = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ) - .await; + let acquired = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ); assert!(acquired.is_some(), "clearing reservation should unblock the provider"); } @@ -2251,21 +3700,19 @@ mod tests { // Low-priority user connects (priority 5 = lower importance) let low_alloc = manager .acquire_connection(&input_name, &low_prio_addr, 5, ConnectionKind::Normal) - .await .expect("low-priority user should get connection"); assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); // Provider is now exhausted - assert!(manager.is_exhausted(&input_name).await); + assert!(manager.is_exhausted(&input_name)); // High-priority user arrives (priority -1 = higher importance), should preempt low-priority user let high_alloc = manager .acquire_connection_with_grace(&input_name, &high_prio_addr, false, -1, ConnectionKind::Normal) - .await .expect("high-priority user should preempt low-priority user and get connection"); assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.release_connection(&high_prio_addr).await; + manager.release_connection(&high_prio_addr); } #[tokio::test] @@ -2282,26 +3729,23 @@ mod tests { // User 1 connects with priority 0 let alloc1 = manager .acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("user1 should get connection"); assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); // Provider is now exhausted - assert!(manager.is_exhausted(&input_name).await); + assert!(manager.is_exhausted(&input_name)); // User 2 arrives with the same priority 0 β€” should NOT preempt user 1 - let alloc2 = manager - .acquire_connection_with_grace( - &input_name, - &user_2_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - ) - .await; + let alloc2 = manager.acquire_connection_with_grace( + &input_name, + &user_2_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + ); assert!(alloc2.is_none(), "same-priority user should not preempt existing user"); - manager.release_connection(&user_1_addr).await; + manager.release_connection(&user_1_addr); } #[tokio::test] @@ -2318,19 +3762,18 @@ mod tests { // High-priority user connects (priority -10) let alloc1 = manager .acquire_connection(&input_name, &high_prio_addr, -10, ConnectionKind::Normal) - .await .expect("high-priority user should get connection"); assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); // Provider is now exhausted - assert!(manager.is_exhausted(&input_name).await); + assert!(manager.is_exhausted(&input_name)); // Low-priority user arrives (priority 10) β€” should NOT preempt high-priority user let alloc2 = - manager.acquire_connection_with_grace(&input_name, &low_prio_addr, false, 10, ConnectionKind::Normal).await; + manager.acquire_connection_with_grace(&input_name, &low_prio_addr, false, 10, ConnectionKind::Normal); assert!(alloc2.is_none(), "low-priority user should not preempt high-priority user"); - manager.release_connection(&high_prio_addr).await; + manager.release_connection(&high_prio_addr); } #[tokio::test] @@ -2348,19 +3791,17 @@ mod tests { // Low-priority user connects (priority 20 = low importance) let low_alloc = manager .acquire_connection(&input_name, &low_prio_addr, 20, ConnectionKind::Normal) - .await .expect("low-priority user should get connection"); assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); let low_token = low_alloc.cancel_token.clone().expect("must have cancel token"); // Provider is now exhausted - assert!(manager.is_exhausted(&input_name).await); + assert!(manager.is_exhausted(&input_name)); // High-priority user arrives WITH grace allowed (default streaming path) // This should get a GracePeriod allocation and then evict the low-prio user let high_alloc = manager .acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal) - .await .expect("high-priority user should get grace allocation and evict low-prio"); assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); @@ -2368,9 +3809,9 @@ mod tests { assert!(low_token.is_cancelled(), "low-prio user should be cancelled after eviction"); // Provider should not be over limit (eviction freed a slot) - assert!(!manager.is_over_limit(&input_name).await, "provider should not be over limit after eviction"); + assert!(!manager.is_over_limit(&input_name), "provider should not be over limit after eviction"); - manager.release_connection(&high_prio_addr).await; + manager.release_connection(&high_prio_addr); } #[tokio::test] @@ -2386,28 +3827,26 @@ mod tests { // User 1 connects with priority 0 let alloc1 = manager .acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("user1 should get connection"); assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); let token1 = alloc1.cancel_token.clone().expect("must have cancel token"); // Provider is now exhausted - assert!(manager.is_exhausted(&input_name).await); + assert!(manager.is_exhausted(&input_name)); // User 2 arrives with the same priority and should be granted grace instead of being rejected. let alloc2 = manager .acquire_connection(&input_name, &user_2_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("same-priority user should get grace allocation"); assert!(matches!(alloc2.allocation, ProviderAllocation::GracePeriod(_))); assert_eq!(alloc2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); // User 1 should NOT be cancelled, and the provider should be temporarily over limit. assert!(!token1.is_cancelled(), "same-prio user should not be evicted"); - assert!(manager.is_over_limit(&input_name).await, "provider should be temporarily over limit during grace"); + assert!(manager.is_over_limit(&input_name), "provider should be temporarily over limit during grace"); - manager.release_connection(&user_1_addr).await; - manager.release_connection(&user_2_addr).await; + manager.release_connection(&user_1_addr); + manager.release_connection(&user_2_addr); } #[tokio::test] @@ -2432,7 +3871,6 @@ mod tests { 20, ConnectionKind::Normal, ) - .await .expect("old low-priority allocation should succeed"); let old_token = old_alloc.cancel_token.clone().expect("old allocation should have cancel token"); assert_eq!(old_alloc.allocation.get_provider_name().as_deref(), Some("provider_2")); @@ -2445,27 +3883,23 @@ mod tests { 20, ConnectionKind::Normal, ) - .await .expect("new low-priority allocation should succeed"); let new_token = new_alloc.cancel_token.clone().expect("new allocation should have cancel token"); assert_eq!(new_alloc.allocation.get_provider_name().as_deref(), Some("provider_1")); { - let mut connections = manager.connections.write().await; + let mut connections = manager.write_connections(); let old_created_at = connections .single - .get(&old_low_addr) - .and_then(|per_addr| per_addr.get(&old_alloc.allocation_id)) + .get(&old_alloc.allocation_id) .map(|info| info.created_at) .expect("old allocation should still be registered"); let (new_created_at, new_priority) = { - let per_addr = connections + let info = connections .single - .get_mut(&new_low_addr) - .expect("new allocation address should still be registered"); - let info = - per_addr.get_mut(&new_alloc.allocation_id).expect("new allocation should still be registered"); + .get_mut(&new_alloc.allocation_id) + .expect("new allocation should still be registered"); let original_created_at = info.created_at; info.created_at = old_created_at; (original_created_at, info.priority) @@ -2480,22 +3914,68 @@ mod tests { tree.insert((new_priority, std::cmp::Reverse(old_created_at), new_alloc.allocation_id), owner); } - assert!(manager.is_exhausted(&input_name).await); + assert!(manager.is_exhausted(&input_name)); // Higher-priority request should now select the first-inserted victim because // priority and created_at are exactly tied across provider aliases. let high_alloc = manager .acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal) - .await .expect("higher-priority request should preempt the first-inserted low-priority victim on exact tie"); assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some("provider_2")); assert!(old_token.is_cancelled(), "first-inserted low-priority victim should be canceled first on exact tie"); assert!(!new_token.is_cancelled(), "later allocation should remain active after the stable tie-break"); - manager.release_connection(&high_prio_addr).await; - manager.release_connection(&old_low_addr).await; - manager.release_connection(&new_low_addr).await; + manager.release_connection(&high_prio_addr); + manager.release_connection(&old_low_addr); + manager.release_connection(&new_low_addr); + } + + #[tokio::test] + async fn shared_promotion_and_release_preserve_other_allocations_on_same_socket( + ) -> Result<(), Box> { + let app_cfg = build_test_app_config(None, 3); + let events = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &events); + let addr = "127.0.0.1:48500".parse()?; + let input = "provider_1".intern(); + let unrelated = manager + .acquire_connection(&input, &addr, 0, ConnectionKind::Normal) + .ok_or("unrelated allocation missing")?; + let origin = + manager.acquire_connection(&input, &addr, 5, ConnectionKind::Normal).ok_or("origin allocation missing")?; + let first = SharedSubscriberId::from_stream_uid(1); + let second = SharedSubscriberId::from_stream_uid(2); + let key = "https://example.invalid/shared.ts"; + assert!(manager.make_shared_connection(&origin, key, first)); + assert_eq!(manager.get_provider_connections_count(), 2); + assert!(manager.read_connections().single.contains_key(&unrelated.allocation_id)); + assert!(manager + .connections + .read() + .unwrap() + .single_by_addr + .get(&addr) + .is_some_and(|allocs| allocs.contains(&unrelated.allocation_id))); + assert!(!unrelated.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + manager.add_shared_connection(&addr, second, key, 9, ConnectionKind::Soft)?; + assert!(manager.reclassify_shared_connection(second, ConnectionKind::Normal, 1)); + { + let connections = manager.read_connections(); + let shared = connections.shared.by_key.get(key).ok_or("shared origin missing")?; + assert_eq!(shared.connections.get(&first).map(|subscriber| subscriber.priority), Some(5)); + assert_eq!(shared.priority, 1); + } + manager.release_shared_connection(second); + manager.release_shared_connection(second); + assert_eq!(manager.get_provider_connections_count(), 2); + assert_eq!(manager.read_connections().shared.by_key.get(key).map(|shared| shared.priority), Some(5)); + manager.release_shared_connection(first); + manager.release_handle(&origin); + assert_eq!(manager.get_provider_connections_count(), 1); + manager.release_handle(&unrelated); + assert_eq!(manager.get_provider_connections_count(), 0); + Ok(()) } #[tokio::test] @@ -2512,19 +3992,24 @@ mod tests { // A starts shared stream with high importance (priority 0). let alloc_a = manager .acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal) - .await .expect("A should get initial connection"); let shared_token = alloc_a.cancel_token.clone().expect("shared allocation should have cancel token"); - manager.make_shared_connection(&addr_a, stream_key).await; + assert!(manager.make_shared_connection(&alloc_a, stream_key, SharedSubscriberId::from_stream_uid(1))); // B joins the same shared stream with lower importance (priority 1). - let join_result = manager.add_shared_connection(&addr_b, stream_key, 1, ConnectionKind::Normal).await; + let join_result = manager.add_shared_connection( + &addr_b, + SharedSubscriberId::from_stream_uid(2), + stream_key, + 1, + ConnectionKind::Normal, + ); assert!(join_result.is_ok(), "B should join existing shared stream, got: {join_result:?}"); // A leaves shared stream. Shared allocation should now inherit B's lower priority. - manager.release_connection(&addr_a).await; + manager.release_connection(&addr_a); { - let connections = manager.connections.read().await; + let connections = manager.read_connections(); let shared = connections.shared.by_key.get(stream_key).expect("shared entry should remain for B"); assert_eq!(shared.priority, 1, "shared priority must downgrade to remaining subscriber priority"); } @@ -2532,14 +4017,13 @@ mod tests { // A starts another stream with higher importance and should preempt B's shared stream. let alloc_a2 = manager .acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal) - .await .expect("A should preempt lower-priority shared stream"); assert_eq!(alloc_a2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted"); - assert!(!manager.is_over_limit(&input_name).await, "provider should not remain over limit after preemption"); + assert!(!manager.is_over_limit(&input_name), "provider should not remain over limit after preemption"); - manager.release_connection(&addr_a).await; - manager.release_connection(&addr_b).await; + manager.release_connection(&addr_a); + manager.release_connection(&addr_b); } #[tokio::test] @@ -2556,34 +4040,37 @@ mod tests { let shared_alloc = manager .acquire_connection(&input_name, &addr_a, 5, ConnectionKind::Normal) - .await .expect("low-priority shared stream should get initial connection"); let shared_token = shared_alloc.cancel_token.clone().expect("shared allocation should have cancel token"); - manager.make_shared_connection(&addr_a, stream_key).await; + assert!(manager.make_shared_connection(&shared_alloc, stream_key, SharedSubscriberId::from_stream_uid(1))); manager - .add_shared_connection(&addr_b, stream_key, 6, ConnectionKind::Normal) - .await + .add_shared_connection( + &addr_b, + SharedSubscriberId::from_stream_uid(2), + stream_key, + 6, + ConnectionKind::Normal, + ) .expect("second subscriber should join shared stream"); let high_alloc = manager .acquire_connection(&input_name, &addr_high, 0, ConnectionKind::Normal) - .await .expect("higher-priority user should preempt lower-priority shared stream"); assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted"); { - let connections = manager.connections.read().await; + let connections = manager.read_connections(); assert!( !connections.shared.by_key.contains_key(stream_key), "preempted shared stream must be removed even with multiple subscribers" ); } - manager.release_connection(&addr_high).await; - manager.release_connection(&addr_a).await; - manager.release_connection(&addr_b).await; + manager.release_connection(&addr_high); + manager.release_connection(&addr_a); + manager.release_connection(&addr_b); } #[tokio::test] @@ -2598,12 +4085,11 @@ mod tests { let addr_b: SocketAddr = "127.0.0.1:49002".parse().unwrap(); // Add connection A (low priority) - let alloc_a = - manager.acquire_connection(&input_name, &addr_a, 10, ConnectionKind::Normal).await.expect("alloc_a"); + let alloc_a = manager.acquire_connection(&input_name, &addr_a, 10, ConnectionKind::Normal).expect("alloc_a"); // Check index has 1 entry { - let connections = manager.connections.read().await; + let connections = manager.read_connections(); let tree = connections.priority_index.get(&input_name).expect("index for provider_1"); assert_eq!(tree.len(), 1, "index should have 1 entry after first allocation"); } @@ -2611,12 +4097,11 @@ mod tests { // High-priority user evicts low-priority via grace path let alloc_b = manager .acquire_connection(&input_name, &addr_b, -5, ConnectionKind::Normal) - .await .expect("alloc_b should evict alloc_a"); // Check index: should have 1 entry (alloc_a evicted, alloc_b added) { - let connections = manager.connections.read().await; + let connections = manager.read_connections(); let tree = connections.priority_index.get(&input_name).expect("index for provider_1"); assert_eq!(tree.len(), 1, "index should have 1 entry after eviction + new allocation"); // The remaining entry should be alloc_b @@ -2625,18 +4110,176 @@ mod tests { } // Release alloc_b - manager.release_handle(&alloc_b).await; + manager.release_handle(&alloc_b); // Check index: should be empty { - let connections = manager.connections.read().await; + let connections = manager.read_connections(); let tree = connections.priority_index.get(&input_name); let is_empty = tree.is_none_or(std::collections::BTreeMap::is_empty); assert!(is_empty, "index should be empty after releasing all connections"); } // Verify alloc_a handle can be safely released (already evicted - no-op) - manager.release_handle(&alloc_a).await; + manager.release_handle(&alloc_a); + } + + #[tokio::test] + async fn test_preemption_respects_registered_body_owner_completion() { + let app_cfg = build_test_app_config(None, 1); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_1: SocketAddr = "127.0.0.1:49061".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:49062".parse().unwrap(); + + // 1. First request with prio 5 occupies limit=1 + let handle_1 = manager + .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) + .expect("first allocation should succeed"); + + // 2. Register body owner on handle_1 + manager.register_body_owner(handle_1.allocation_id); + + // 3. Higher-priority request with prio -1 arrives without grace while handle_1 is active + let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); + + // 4. Replacement must NOT be admitted before completion + assert!(handle_2.is_none(), "replacement must not be admitted before victim completion"); + assert!( + handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), + "victim should receive cancel signal" + ); + assert_eq!( + handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), + tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, + "victim close reason must be PriorityPreempted" + ); + + // 5. Victim completes upstream reading + handle_1.completion_token.as_ref().expect("completion token").cancel(); + + // Allow background complete_release task to run + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(20)).await; + + // 6. Now replacement request can be admitted + let handle_2 = manager + .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) + .expect("replacement should succeed after victim has completed release"); + + manager.release_handle(&handle_2); + } + + #[tokio::test] + async fn test_preemption_respects_opening_connection_completion() { + let app_cfg = build_test_app_config(None, 1); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_1: SocketAddr = "127.0.0.1:49063".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:49064".parse().unwrap(); + + // 1. First request with prio 5 occupies limit=1 + let handle_1 = manager + .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) + .expect("first allocation should succeed"); + manager.mark_opening(handle_1.allocation_id); + + // Verify lifecycle is Opening and body owner is not yet registered + { + let connections = manager.read_connections(); + let info = connections.single.get(&handle_1.allocation_id).expect("handle_1 exists"); + assert_eq!(info.lifecycle, tuliprox_core::model::ConnectionLifecycle::Opening); + assert!(!info.has_body_owner); + } + + // 2. Higher-priority request with prio -1 arrives without grace while handle_1 is in Opening + let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); + + // 3. Replacement must NOT be admitted before completion of opening + assert!(handle_2.is_none(), "replacement must not be admitted while victim is still opening"); + assert!( + handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), + "victim should receive cancel signal during open" + ); + assert_eq!( + handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), + tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, + "victim close reason must be PriorityPreempted" + ); + + // 4. Victim's open future completes and signals completion + handle_1.completion_token.as_ref().expect("completion token").cancel(); + + // Allow background complete_release task to run + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(20)).await; + + // 5. Now replacement request can be admitted + let handle_2 = manager + .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) + .expect("replacement should succeed after victim open future has completed release"); + + manager.release_handle(&handle_2); + } + + #[tokio::test] + async fn test_owner_index_consistent_through_lifecycle() { + let app_cfg = build_test_app_config(None, 2); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_a: SocketAddr = "127.0.0.1:49051".parse().unwrap(); + let addr_b: SocketAddr = "127.0.0.1:49052".parse().unwrap(); + + let owner_a = "owner-a"; + let owner_b = "owner-b"; + let alloc_a = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_a, + false, + 0, + ConnectionKind::Normal, + Some(owner_a), + ) + .expect("alloc_a"); + let alloc_b = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_b, + false, + 0, + ConnectionKind::Normal, + Some(owner_b), + ) + .expect("alloc_b"); + + { + let connections = manager.read_connections(); + assert_eq!(connections.by_owner.get(owner_a).map_or(0, HashSet::len), 1, "owner-a has one allocation"); + assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b has one allocation"); + assert!(connections.by_owner.get(owner_a).unwrap().contains(&alloc_a.allocation_id)); + assert!(connections.by_owner.get(owner_b).unwrap().contains(&alloc_b.allocation_id)); + } + + manager.release_handle(&alloc_a); + + { + let connections = manager.read_connections(); + assert!(!connections.by_owner.contains_key(owner_a), "owner-a index is removed after release"); + assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b still indexed"); + } + + manager.release_handle(&alloc_b); + { + let connections = manager.read_connections(); + assert!(connections.by_owner.is_empty(), "owner index is empty after all releases"); + } } #[tokio::test] @@ -2651,20 +4294,18 @@ mod tests { let soft_alloc = manager .acquire_connection(&input_name, &soft_addr, default_user_priority(), ConnectionKind::Soft) - .await .expect("soft allocation"); let soft_token = soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); let normal_alloc = manager .acquire_connection(&input_name, &normal_addr, default_user_priority(), ConnectionKind::Normal) - .await .expect("normal allocation should preempt soft allocation"); tokio::task::yield_now().await; assert!(soft_token.is_cancelled(), "soft allocation should be preempted by normal traffic"); - assert_eq!(manager.get_provider_connections_count().await, 1); + assert_eq!(manager.get_provider_connections_count(), 1); - manager.release_handle(&normal_alloc).await; + manager.release_handle(&normal_alloc); } #[tokio::test] @@ -2679,20 +4320,18 @@ mod tests { let low_soft_alloc = manager .acquire_connection(&input_name, &low_soft_addr, 10, ConnectionKind::Soft) - .await .expect("low-priority soft allocation"); let low_soft_token = low_soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); let high_soft_alloc = manager .acquire_connection(&input_name, &high_soft_addr, -5, ConnectionKind::Soft) - .await .expect("higher-priority soft allocation should preempt lower-priority soft allocation"); tokio::task::yield_now().await; assert!(low_soft_token.is_cancelled(), "lower-priority soft allocation should be preempted"); - assert_eq!(manager.get_provider_connections_count().await, 1); + assert_eq!(manager.get_provider_connections_count(), 1); - manager.release_handle(&high_soft_alloc).await; + manager.release_handle(&high_soft_alloc); } #[tokio::test] @@ -2707,28 +4346,782 @@ mod tests { let promoted_alloc = manager .acquire_connection(&input_name, &promoted_addr, default_user_priority(), ConnectionKind::Soft) - .await .expect("initial soft allocation"); let promoted_token = promoted_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); assert!( - manager.reclassify_connection(&promoted_addr, ConnectionKind::Normal, default_user_priority()).await, + manager.reclassify_connection(&promoted_addr, ConnectionKind::Normal, default_user_priority()), "soft allocation should be promotable to normal" ); - let challenger = manager - .acquire_connection_with_grace( + let challenger = manager.acquire_connection_with_grace( + &input_name, + &challenger_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + ); + assert!(challenger.is_none(), "same-priority normal traffic should not preempt a promoted normal connection"); + assert!(!promoted_token.is_cancelled(), "promoted connection should remain active"); + assert_eq!(manager.get_provider_connections_count(), 1); + + manager.release_handle(&promoted_alloc); + } + + #[tokio::test] + async fn test_reclassify_connection_for_owner_preserves_other_owner_on_same_socket() { + let app_cfg = build_test_app_config(None, 2); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let shared_addr: SocketAddr = "127.0.0.1:49020".parse().unwrap(); + + let alloc_1 = manager + .acquire_connection_with_lease_for_session( &input_name, - &challenger_addr, + &shared_addr, + false, + -10, + ConnectionKind::Soft, + Some(PlaybackLeaseRef::new("owner-1", PlaybackKind::LiveTs)), + ) + .expect("first soft allocation"); + + let alloc_2 = manager + .acquire_connection_with_lease_for_session( + &input_name, + &shared_addr, + false, + -10, + ConnectionKind::Soft, + Some(PlaybackLeaseRef::new("owner-2", PlaybackKind::LiveTs)), + ) + .expect("second soft allocation"); + + assert!(manager.reclassify_connection_for_owner(&shared_addr, Some("owner-1"), ConnectionKind::Normal, 0)); + + { + let connections = manager.read_connections(); + let info_1 = connections.single.get(&alloc_1.allocation_id).expect("alloc_1 exists"); + assert_eq!(info_1.kind, ConnectionKind::Normal); + assert_eq!(info_1.priority, 0); + + let info_2 = connections.single.get(&alloc_2.allocation_id).expect("alloc_2 exists"); + assert_eq!(info_2.kind, ConnectionKind::Soft); + assert_eq!(info_2.priority, -10); + } + + manager.release_handle(&alloc_1); + manager.release_handle(&alloc_2); + } + + /// A=2 / B=3 pool: five confirmed playbacks exhaust the pool and a sixth start is + /// rejected; releasing one confirmed playback frees exactly one slot again. + #[tokio::test] + async fn sixth_playback_rejected_after_pool_exhausted_by_confirmed_leases() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let mut acquired = Vec::new(); + for index in 0..5u16 { + let owner = format!("playback-{index}"); + let addr = SocketAddr::from(([172, 18, 0, 9], 60_000 + index)); + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(&owner), + ) + .expect("pool has capacity for five playbacks"); + manager.confirm_playback_activity(&owner); + acquired.push((handle, owner)); + } + + let sixth = manager.acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 60_005)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("playback-5"), + ); + assert!(sixth.is_none(), "sixth start must be rejected once the pool is exhausted"); + + manager.release_handle(&acquired[0].0); + let replacement = manager.acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 60_006)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("playback-6"), + ); + assert!(replacement.is_some(), "freed capacity must be selectable again"); + manager.release_handle(&replacement.expect("replacement handle")); + for (handle, _owner) in &acquired[1..] { + manager.release_handle(handle); + } + assert_eq!(manager.get_provider_connections_count(), 0); + } + + /// Confirmed reconnect-capable lease, physical handle release first, then the real + /// provider-error outcome: the error must reach the lease and leave no idle reserve. + #[tokio::test(start_paused = true)] + async fn provider_error_cleanup_after_physical_release_does_not_keep_idle_lease() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "r5-error-owner"; + let addr = SocketAddr::from(([172, 18, 0, 9], 55_000)); + let handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &addr, false, default_user_priority(), ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), ) - .await; - assert!(challenger.is_none(), "same-priority normal traffic should not preempt a promoted normal connection"); - assert!(!promoted_token.is_cancelled(), "promoted connection should remain active"); - assert_eq!(manager.get_provider_connections_count().await, 1); + .expect("acquire a reconnect-capable slot"); + let request_id = handle.playback_request_id.expect("identified request id"); + manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15); + manager.confirm_playback_activity(owner); - manager.release_handle(&promoted_alloc).await; + // Physical release must not conclude the request; the outcome is decided later. + manager.release_handle(&handle); + assert_eq!( + manager.provider_lease_usage(&input_name).active, + 1, + "physical release must not finish a confirmed lease" + ); + + manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed); + assert_eq!( + manager.provider_lease_usage(&input_name).total(), + 0, + "provider error must not keep an idle reconnect lease" + ); + } + + #[tokio::test(start_paused = true)] + async fn stale_refresh_does_not_recreate_terminal_request() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "terminal-refresh-owner"; + let handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 55_002)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), + ) + .expect("acquire identified request"); + let request_id = handle.playback_request_id.expect("identified request id"); + let lease_ref = PlaybackLeaseRef { owner, kind: PlaybackKind::LiveHls, request_id }; + + manager.release_handle(&handle); + manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed); + assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); + + manager.refresh_playback_lease(&input_name, &lease_ref, 15); + manager.confirm_identified_playback_activity(owner, request_id); + assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); + } + + /// Counterexample: the same sequence with a clean end keeps the configured idle window. + #[tokio::test(start_paused = true)] + async fn clean_cleanup_after_physical_release_keeps_configured_idle_window() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "r5-clean-owner"; + let addr = SocketAddr::from(([172, 18, 0, 9], 55_001)); + + let handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), + ) + .expect("acquire a reconnect-capable slot"); + let request_id = handle.playback_request_id.expect("identified request id"); + manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15); + manager.confirm_playback_activity(owner); + + manager.release_handle(&handle); + manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::Completed); + let usage = manager.provider_lease_usage(&input_name); + assert_eq!(usage.active, 0, "clean end must move the lease out of active"); + assert_eq!(usage.idle, 1, "clean end must keep the configured reconnect window"); + } + + /// max=1: X fetch β†’ physical release β†’ Y acquire β†’ X first byte within the startup + /// deadline. X's late confirmation records media but must not over-commit Y's slot. + #[tokio::test(start_paused = true)] + async fn late_first_byte_cannot_reserve_slot_taken_by_other_playback() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let x_handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 40_000)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)), + ) + .expect("x acquires the only slot"); + manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15); + manager.release_handle(&x_handle); + + let y_handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 40_001)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("y", PlaybackKind::LiveHls)), + ) + .expect("y acquires the now-free slot"); + + manager.confirm_playback_activity("x"); + assert_eq!(manager.get_provider_connections_count(), 1, "y still holds the only slot"); + assert!( + !manager.is_provider_reserved_for_other_session(&input_name, Some("y")), + "x's late confirmation must not over-commit the provider" + ); + + manager.confirm_playback_activity("y"); + manager.release_handle(&y_handle); + manager.finish_playback_request("y", PlaybackRequestOutcome::Completed, None); + } + + /// Counterexample: the same X fetch β†’ release β†’ first-byte sequence without a + /// competing Y keeps the free slot as a real reservation. + #[tokio::test(start_paused = true)] + async fn first_byte_reserves_when_slot_still_free() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let x_handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 40_002)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)), + ) + .expect("x acquires the only slot"); + manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15); + manager.release_handle(&x_handle); + + // No competing playback took the slot, so X's first byte reserves it. + manager.confirm_playback_activity("x"); + assert!( + manager.is_provider_reserved_for_other_session(&input_name, Some("other")), + "a free slot must be reservable by the confirming playback" + ); + + manager.finish_playback_request("x", PlaybackRequestOutcome::Completed, None); + } + + /// The RAII owner releases the allocation synchronously even when dropped outside + /// a tokio runtime, so a body/context drop can never leak a provider slot. + #[test] + fn managed_handle_drop_outside_runtime_releases_allocation() { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let (manager, managed_handle) = rt.block_on(async { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name = "provider_1".intern(); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 45_000)), + false, + 0, + ConnectionKind::Normal, + Some("managed-owner"), + ) + .expect("allocation should succeed"); + assert_eq!(manager.get_provider_connections_count(), 1); + + let managed_handle = super::ManagedProviderHandle::new(Arc::clone(&manager), handle); + (manager, managed_handle) + }); + + drop(managed_handle); + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] + fn latency_percentile(sorted: &[u64], p: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + let index = ((sorted.len() as f64 - 1.0) * p).round() as usize; + sorted[index.min(sorted.len() - 1)] + } + + #[cfg(target_os = "linux")] + fn resident_set_kib() -> Option { + let Ok(content) = std::fs::read_to_string("/proc/self/status") else { + return None; + }; + for line in content.lines() { + if let Some(rest) = line.strip_prefix("VmRSS:") { + return rest.split_whitespace().next().and_then(|v| v.parse::().ok()); + } + } + None + } + + #[cfg(not(target_os = "linux"))] + fn resident_set_kib() -> Option { None } + + struct LeaseWorkloadStats { + ops: u64, + elapsed_secs: f64, + acquire_p50: u64, + acquire_p95: u64, + acquire_p99: u64, + confirm_p50: u64, + confirm_p95: u64, + confirm_p99: u64, + release_p50: u64, + release_p95: u64, + release_p99: u64, + } + + #[allow(clippy::cast_possible_truncation)] + async fn measure_lease_workload( + manager: &Arc, + input_name: &Arc, + concurrency: usize, + rounds_per_task: usize, + ) -> LeaseWorkloadStats { + let samples = Arc::new(std::sync::Mutex::new(Vec::<(u64, u64, u64)>::new())); + let started = std::time::Instant::now(); + let mut tasks = tokio::task::JoinSet::new(); + for task_index in 0..concurrency { + let manager = Arc::clone(manager); + let input_name = Arc::clone(input_name); + let samples = Arc::clone(&samples); + tasks.spawn(async move { + for round in 0..rounds_per_task { + let owner = format!("bench-{task_index}-{round}"); + let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + (task_index as u16))); + let acquire_at = std::time::Instant::now(); + let Some(handle) = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some(&owner), + ) else { + continue; + }; + let acquire_us = acquire_at.elapsed().as_micros() as u64; + + let confirm_at = std::time::Instant::now(); + manager.confirm_playback_activity(&owner); + let confirm_us = confirm_at.elapsed().as_micros() as u64; + + let release_at = std::time::Instant::now(); + manager.release_handle(&handle); + let release_us = release_at.elapsed().as_micros() as u64; + + // A physical release never concludes the request; model the real + // lifecycle by finishing with a clean outcome so the lease returns + // to baseline (LiveTs is not reconnect-capable, so it is removed). + manager.finish_playback_request(&owner, PlaybackRequestOutcome::Completed, None); + + samples.lock().unwrap().push((acquire_us, confirm_us, release_us)); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.expect("benchmark task must not fail"); + } + + let elapsed_secs = started.elapsed().as_secs_f64(); + let samples = Arc::try_unwrap(samples).expect("samples unique").into_inner().unwrap(); + let mut acquire = Vec::with_capacity(samples.len()); + let mut confirm = Vec::with_capacity(samples.len()); + let mut release = Vec::with_capacity(samples.len()); + for (a, c, r) in samples { + acquire.push(a); + confirm.push(c); + release.push(r); + } + acquire.sort_unstable(); + confirm.sort_unstable(); + release.sort_unstable(); + let ops = acquire.len() as u64; + LeaseWorkloadStats { + ops, + elapsed_secs, + acquire_p50: latency_percentile(&acquire, 0.50), + acquire_p95: latency_percentile(&acquire, 0.95), + acquire_p99: latency_percentile(&acquire, 0.99), + confirm_p50: latency_percentile(&confirm, 0.50), + confirm_p95: latency_percentile(&confirm, 0.95), + confirm_p99: latency_percentile(&confirm, 0.99), + release_p50: latency_percentile(&release, 0.50), + release_p95: latency_percentile(&release, 0.95), + release_p99: latency_percentile(&release, 0.99), + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[ignore = "manager microbenchmark: cargo +stable test -p tuliprox-session --release -- --ignored --nocapture bench_provider_lease_microbenchmark"] + #[allow(clippy::cast_precision_loss)] + async fn bench_provider_lease_microbenchmark() { + let manager = Arc::new(ActiveProviderManager::new( + &create_test_app_config_single_unlimited_provider_pool(), + &Arc::new(EventManager::new()), + )); + let input_name: Arc = "provider_1".intern(); + + // Warm-up before measurement so allocator and lock caches are exercised. + for index in 0..64u16 { + let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + index)); + let Some(handle) = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + None, + ) else { + continue; + }; + manager.release_handle(&handle); + } + + let baseline_rss_kib = resident_set_kib(); + eprintln!("provider lease manager microbenchmark (unlimited provider, in-process, no HTTP bodies)"); + for concurrency in [1usize, 5, 50, 200] { + let stats = measure_lease_workload(&manager, &input_name, concurrency, 10).await; + assert_eq!( + stats.ops, + (concurrency * 10) as u64, + "unexpected acquire failures at concurrency {concurrency}" + ); + let throughput = stats.ops as f64 / stats.elapsed_secs.max(f64::EPSILON); + let rss_delta_kib = resident_set_kib() + .zip(baseline_rss_kib) + .map_or_else(|| "unsupported".to_string(), |(rss, baseline)| rss.saturating_sub(baseline).to_string()); + eprintln!( + "concurrency={concurrency:>3} ops={:>5} throughput={:>9.1} ops/s | acquire p50/p95/p99={}/{}/{}us | confirm p50/p95/p99={}/{}/{}us | release p50/p95/p99={}/{}/{}us | rss_delta_kib={}", + stats.ops, + throughput, + stats.acquire_p50, + stats.acquire_p95, + stats.acquire_p99, + stats.confirm_p50, + stats.confirm_p95, + stats.confirm_p99, + stats.release_p50, + stats.release_p95, + stats.release_p99, + rss_delta_kib, + ); + } + + // Soak-style churn: every connection and lease must return to baseline. + assert_eq!(manager.get_provider_connections_count(), 0); + let usage = manager.provider_lease_usage(&input_name); + assert_eq!(usage.total(), 0, "lease table must return to baseline after churn"); + } + + #[tokio::test] + async fn closing_state_blocks_slot_release_until_completion_token_cancelled() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let handle1 = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_001)), + false, + 0, + ConnectionKind::Normal, + Some("session-1"), + ) + .expect("first allocation should succeed"); + assert_eq!(manager.get_provider_connections_count(), 1); + + let completion_token = handle1.completion_token.clone().expect("completion token present"); + let alloc_id = handle1.allocation_id; + + // Mark Closing (simulating supersede / release_playback_connections_await) + manager.mark_closing(alloc_id); + + // Drop the handle while completion_token is still pending (upstream body not yet closed) + manager.release_handle(&handle1); + + // The slot must still be occupied by the Closing allocation + assert_eq!(manager.get_provider_connections_count(), 1); + + // A second start attempt must fail because the provider is at its limit (1 connection) + let handle2 = manager.acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_002)), + false, + 0, + ConnectionKind::Normal, + Some("session-2"), + ); + assert!(handle2.is_none(), "second start must be rejected while first slot is Closing"); + + // Now signal completion (upstream body owner dropped upstream socket) + completion_token.cancel(); + + // Allow background reaper task to execute complete_release + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!(manager.get_provider_connections_count(), 0); + + // Second start attempt now succeeds + let handle3 = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_003)), + false, + 0, + ConnectionKind::Normal, + Some("session-3"), + ) + .expect("start must succeed after completion releases the slot"); + assert_eq!(manager.get_provider_connections_count(), 1); + + manager.release_handle(&handle3); + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[tokio::test] + async fn reaper_cleanup_is_idempotent_and_removes_all_indices() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 47_001)), + false, + 0, + ConnectionKind::Normal, + Some("session-idempotent"), + ) + .expect("allocation should succeed"); + + let completion_token = handle.completion_token.clone().expect("completion token present"); + let alloc_id = handle.allocation_id; + manager.mark_closing(alloc_id); + + manager.release_handle(&handle); + completion_token.cancel(); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(manager.get_provider_connections_count(), 0); + + // Duplicate releases must be completely safe and no-op + manager.complete_release(alloc_id); + manager.release_handle(&handle); + manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 47_001))); + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[tokio::test] + async fn test_single_request_preemption_awaits_victim_completion() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let low_addr = SocketAddr::from(([127, 0, 0, 1], 48_001)); + let high_addr = SocketAddr::from(([127, 0, 0, 1], 48_002)); + + let low_handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &low_addr, + false, + 10, + ConnectionKind::Normal, + Some("session-victim"), + ) + .expect("initial low-priority allocation succeeds"); + + manager.mark_opening(low_handle.allocation_id); + assert!(manager.register_body_owner(low_handle.allocation_id)); + + let cancel_token = low_handle.cancel_token.clone().unwrap(); + let completion_token = low_handle.completion_token.clone().unwrap(); + + // Simulate upstream body owner: when cancelled, complete release after 30ms + let comp_clone = completion_token.clone(); + tokio::spawn(async move { + cancel_token.cancelled().await; + tokio::time::sleep(Duration::from_millis(30)).await; + comp_clone.cancel(); + }); + + // High priority acquire should preempt the victim, await its completion, and succeed in the same call + let high_handle = manager + .acquire_connection_with_lease_for_session_await( + &input_name, + &high_addr, + false, + 0, + ConnectionKind::Normal, + None, + ) + .await + .expect("high-priority allocation must succeed in the same call after awaiting victim completion"); + + assert_eq!(manager.get_provider_connections_count(), 1); + manager.release_handle(&high_handle); + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[tokio::test] + async fn stale_handle_does_not_cancel_newer_generation_slot() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + let addr = SocketAddr::from(([127, 0, 0, 1], 49_001)); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some("session-gen"), + ) + .expect("initial allocation succeeds"); + + manager.mark_opening(handle.allocation_id); + assert!(manager.register_body_owner(handle.allocation_id)); + + // Gen 0 has an old completion token + let old_completion_token = handle.completion_token.clone().expect("completion token present"); + + // Renew opening tokens to advance generation to 1 + let (_new_cancel, new_completion, gen) = + manager.renew_opening_tokens(handle.allocation_id).expect("renewal should succeed"); + assert_eq!(gen, 1); + + // Simulate an old handle belonging to generation 0 whose completion token was cancelled + old_completion_token.cancel(); + let stale_handle = tuliprox_core::model::ProviderHandle { + playback_request_id: handle.playback_request_id, + binding_tag: handle.binding_tag, + client_id: handle.client_id, + allocation_id: handle.allocation_id, + allocation: handle.allocation.clone(), + cancel_token: handle.cancel_token.clone(), + completion_token: Some(old_completion_token), + close_reason: Arc::clone(&handle.close_reason), + open_generation: 0, // Stale generation + }; + + // Releasing the stale handle must NOT poison/cancel the new generation's completion token + manager.release_handle(&stale_handle); + assert!( + !new_completion.is_cancelled(), + "new generation completion token must remain uncancelled after stale handle release" + ); + assert_eq!(manager.get_provider_connections_count(), 1, "slot must still be held by the active new generation"); + + // When the real new generation completes, the slot is reaped + new_completion.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(manager.get_provider_connections_count(), 0); + } + + #[tokio::test] + async fn dropped_cleanup_future_does_not_permanently_leak_slot() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + let addr = SocketAddr::from(([127, 0, 0, 1], 49_002)); + let addrs = [addr]; + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some("session-drop"), + ) + .expect("allocation succeeds"); + + manager.mark_opening(handle.allocation_id); + assert!(manager.register_body_owner(handle.allocation_id)); + let completion_token = handle.completion_token.clone().expect("token present"); + + // Start release_playback_connections_await but drop it before completion + { + let cleanup_fut = manager.release_playback_connections_await("session-drop", &addrs); + // Poll once and drop + tokio::select! { + biased; + () = async {} => {}, + () = cleanup_fut => {}, + } + } + + // Slot must still be occupied + assert_eq!(manager.get_provider_connections_count(), 1); + + // Now the handle is released and the body owner signals completion + manager.release_handle(&handle); + completion_token.cancel(); + + // Give the background reaper task time to run + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!( + manager.get_provider_connections_count(), + 0, + "slot must be freed after completion and not permanently leaked in Closing state" + ); } } diff --git a/backend/session/src/active_user_manager/mod.rs b/backend/session/src/active_user_manager/mod.rs index 97a4d9148..31564e5dc 100644 --- a/backend/session/src/active_user_manager/mod.rs +++ b/backend/session/src/active_user_manager/mod.rs @@ -1,5 +1,8 @@ use crate::{ - active_provider_manager::ConnectionKind, connection_manager::CleanupEvent, ActiveProviderManager, EventManager, + active_provider_manager::ConnectionKind, + connection_manager::CleanupEvent, + stream::{uses_direct_body_idle_timeout, DIRECT_BODY_IDLE_TIMEOUT_SECS}, + ActiveProviderManager, EventManager, }; use arc_swap::ArcSwapOption; use jsonwebtoken::get_current_timestamp; @@ -152,10 +155,54 @@ struct UserConnectionCounts { soft: u16, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdmissionRejectionReason { + UserConnectionsExhausted, + RecentEvictionReentry, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ConnectionAdmission { - pub permission: UserConnectionPermission, - pub kind: Option, + permission: UserConnectionPermission, + kind: Option, + rejection_reason: Option, +} + +impl ConnectionAdmission { + pub fn allowed(kind: Option) -> Self { + Self { permission: UserConnectionPermission::Allowed, kind, rejection_reason: None } + } + + pub fn grace_period(kind: Option) -> Self { + Self { permission: UserConnectionPermission::GracePeriod, kind, rejection_reason: None } + } + + pub fn exhausted(reason: AdmissionRejectionReason, kind: Option) -> Self { + Self { permission: UserConnectionPermission::Exhausted, kind, rejection_reason: Some(reason) } + } + + /// Maps a stored session permission onto an admission. An exhausted session is + /// classified as a genuine connection-limit rejection so the + /// `Exhausted β‡’ rejection_reason.is_some()` invariant always holds. + pub fn from_permission(permission: UserConnectionPermission, kind: Option) -> Self { + match permission { + UserConnectionPermission::Allowed => Self::allowed(kind), + UserConnectionPermission::GracePeriod => Self::grace_period(kind), + UserConnectionPermission::Exhausted => { + Self::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, kind) + } + } + } + + pub fn permission(&self) -> UserConnectionPermission { self.permission } + + pub fn kind(&self) -> Option { self.kind } + + pub fn rejection_reason(&self) -> Option { self.rejection_reason } + + pub fn is_reentry_suppressed(&self) -> bool { + self.rejection_reason == Some(AdmissionRejectionReason::RecentEvictionReentry) + } } #[derive(Debug, Clone, Copy)] @@ -165,6 +212,38 @@ struct PromotionAction { new_priority: i8, } +#[derive(Debug, Clone, Copy)] +struct StreamRequestClaim { + stream_uid: u32, + addr: SocketAddr, +} + +#[derive(Debug)] +pub enum StreamRequestDetach { + NotFound, + Retained(StreamInfo), + Preserved(StreamInfo), + Removed(StreamInfo), +} + +impl StreamRequestDetach { + pub fn request_was_detached(&self) -> bool { !matches!(self, Self::NotFound) } + + pub fn stream_info(&self) -> Option<&StreamInfo> { + match self { + Self::NotFound => None, + Self::Retained(stream) | Self::Preserved(stream) | Self::Removed(stream) => Some(stream), + } + } + + fn into_removed(self) -> Option { + match self { + Self::Removed(stream) => Some(stream), + Self::NotFound | Self::Retained(_) | Self::Preserved(_) => None, + } + } +} + #[derive(Debug)] struct UserConnectionData { max_connections: u32, @@ -175,6 +254,7 @@ struct UserConnectionData { grace_ts: u64, sessions: Vec, streams: Vec, + stream_request_claims: HashMap, stream_kinds: HashMap, stream_normal_priorities: HashMap, ts: u64, @@ -191,6 +271,7 @@ impl UserConnectionData { grace_ts: 0, sessions: Vec::new(), streams: Vec::new(), + stream_request_claims: HashMap::new(), stream_kinds: HashMap::new(), stream_normal_priorities: HashMap::new(), ts: current_time_secs(), @@ -222,6 +303,40 @@ impl UserConnectionData { migrated_addrs } + fn release_addr_from_stream_session(&mut self, addr: &SocketAddr, uid: u32) -> HashMap> { + let Some(token) = + self.streams.iter().find(|stream| stream.uid == uid).and_then(|stream| stream.session_token.as_deref()) + else { + return HashMap::new(); + }; + if self.streams.iter().any(|stream| { + stream.uid != uid + && !stream.preserved + && stream.addr == *addr + && stream.session_token.as_deref() == Some(token) + }) { + return HashMap::new(); + } + let Some(session) = self.sessions.iter_mut().find(|session| session.token == token) else { + return HashMap::new(); + }; + HashMap::from([(session.token.clone(), release_session_addr(session, addr))]) + } + + fn attach_stream_request(&mut self, request_uid: u32, stream_uid: u32, addr: SocketAddr) { + self.stream_request_claims.insert(request_uid, StreamRequestClaim { stream_uid, addr }); + } + + fn detach_stream_request(&mut self, request_uid: u32) -> Option<(u32, bool)> { + let claim = self.stream_request_claims.remove(&request_uid)?; + let is_last = !self.stream_request_claims.values().any(|other| other.stream_uid == claim.stream_uid); + Some((claim.stream_uid, is_last)) + } + + fn remove_stream_request_claims(&mut self, stream_uid: u32) { + self.stream_request_claims.retain(|_, claim| claim.stream_uid != stream_uid); + } + fn increment_kind(&mut self, kind: ConnectionKind) { self.connections = self.connections.saturating_add(1); match kind { @@ -268,6 +383,7 @@ impl UserConnectionData { connection_changed = true; } self.stream_normal_priorities.remove(&uid); + self.remove_stream_request_claims(uid); self.streams.swap_remove(stream_idx); removed_count += 1; } @@ -406,7 +522,7 @@ fn uses_session_reentry_guard(stream: &StreamInfo) -> bool { #[derive(Clone, Copy, Debug)] struct RecentWinnerProtection { protected_addr: SocketAddr, - expires_at: u64, + expires_at: Instant, } #[derive(Debug, Default)] @@ -418,9 +534,9 @@ struct UserConnections { key_by_addr: HashMap, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] struct SocketRegistration { - username: String, + usernames: HashSet, ts: u64, } @@ -520,7 +636,27 @@ fn clear_session_addr(session: &mut UserSession, addr: &SocketAddr) -> bool { } impl SocketRegistration { - fn anonymous() -> Self { Self { username: String::new(), ts: current_time_secs() } } + fn anonymous() -> Self { Self { usernames: HashSet::new(), ts: current_time_secs() } } + + fn add_user(&mut self, username: &str, ts: u64) { + if !username.is_empty() && !self.usernames.contains(username) { + self.usernames.insert(username.to_string()); + } + self.ts = ts; + } + + fn remove_user(&mut self, username: &str) -> bool { + self.usernames.remove(username); + self.usernames.is_empty() + } + + fn is_empty(&self) -> bool { self.usernames.is_empty() } + + fn primary_username(&self) -> Option<&str> { self.usernames.iter().map(String::as_str).min() } + + fn all_usernames(&self) -> Vec { self.usernames.iter().cloned().collect() } + + fn into_usernames(self) -> HashSet { self.usernames } } struct UserSessionParams<'a> { @@ -555,6 +691,7 @@ pub struct ActiveUserManager { provider_manager: tokio::sync::OnceCell>, transition_gates: Mutex>>>, pub dropped_cleanup_events: AtomicU64, + reentry_suppressed_total: AtomicU64, divergence_cache: Mutex>, divergence_cooldown_secs: u64, } @@ -655,11 +792,19 @@ impl ActiveUserManager { provider_manager: tokio::sync::OnceCell::new(), transition_gates: Mutex::new(HashMap::new()), dropped_cleanup_events: AtomicU64::new(0), + reentry_suppressed_total: AtomicU64::new(0), divergence_cache: Mutex::new(LruCache::new(DIVERGENCE_CACHE_CAPACITY)), divergence_cooldown_secs: 300, } } + /// Cumulative count of admission requests quietly suppressed by the recent-eviction + /// reentry guard. A diagnostic counter, not a failure metric. + pub fn reentry_suppressed_total(&self) -> u64 { self.reentry_suppressed_total.load(Ordering::Relaxed) } + + /// Records one suppressed reentry retry. + pub fn record_reentry_suppressed(&self) { self.reentry_suppressed_total.fetch_add(1, Ordering::Relaxed); } + fn transition_gate_key(username: &str, token: &str) -> String { let mut key = String::with_capacity(username.len() + token.len() + 1); key.push_str(username); @@ -721,6 +866,19 @@ impl ActiveUserManager { .collect() } + /// Atomically removes all user requests and sessions during terminal shutdown. + pub async fn drain_for_shutdown(&self) -> Vec { + let connections = std::mem::take(&mut *self.connections.write().await); + self.adaptive_expiry_queue.lock().await.clear(); + self.adaptive_expiry_index.lock().await.clear(); + self.transition_gates.lock().await.clear(); + connections + .by_key + .into_values() + .flat_map(|data| data.streams.into_iter().filter(|stream| !stream.preserved)) + .collect() + } + async fn log_active_user(&self) { let is_log_user_enabled = self.is_log_user_enabled(); let (user_count, user_connection_count) = { self.active_users_and_connections().await }; @@ -741,10 +899,6 @@ impl ActiveUserManager { } async fn emit_promotion_update(&self, username: &str, action: PromotionAction) { - if let Some(provider_manager) = self.provider_manager.get() { - provider_manager.reclassify_connection(&action.addr, ConnectionKind::Normal, action.new_priority).await; - } - let maybe_stream = { let user_connections = self.connections.read().await; user_connections.by_key.get(username).and_then(|connection_data| { @@ -752,6 +906,22 @@ impl ActiveUserManager { }) }; if let Some(stream_info) = maybe_stream { + if let Some(provider_manager) = self.provider_manager.get() { + if stream_info.channel.shared { + provider_manager.reclassify_shared_connection( + tuliprox_core::model::SharedSubscriberId::from_stream_uid(action.uid), + ConnectionKind::Normal, + action.new_priority, + ); + } else { + provider_manager.reclassify_connection_for_owner( + &action.addr, + stream_info.session_token.as_deref(), + ConnectionKind::Normal, + action.new_priority, + ); + } + } self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); } } @@ -761,16 +931,20 @@ impl ActiveUserManager { /// the underlying HTTP connection may still remain open. #[allow(clippy::too_many_lines)] pub async fn release_stream(&self, addr: &SocketAddr) -> Option { - self.release_stream_inner(addr, None).await + self.release_stream_inner(addr, None).await.into_removed() } #[allow(clippy::too_many_lines)] pub async fn release_stream_by_uid(&self, addr: &SocketAddr, stream_uid: u32) -> Option { - self.release_stream_inner(addr, Some(stream_uid)).await + self.release_stream_request_by_uid(addr, stream_uid).await.into_removed() + } + + pub async fn release_stream_request_by_uid(&self, addr: &SocketAddr, request_uid: u32) -> StreamRequestDetach { + self.release_stream_inner(addr, Some(request_uid)).await } #[allow(clippy::too_many_lines)] - async fn release_stream_inner(&self, addr: &SocketAddr, stream_uid: Option) -> Option { + async fn release_stream_inner(&self, addr: &SocketAddr, stream_uid: Option) -> StreamRequestDetach { let ( removed_stream, username, @@ -782,29 +956,39 @@ impl ActiveUserManager { ) = { let mut user_connections = self.connections.write().await; - let username = match stream_uid { - Some(uid) => user_connections.by_key.iter().find_map(|(username, connection_data)| { + let username = if let Some(uid) = stream_uid { + user_connections.by_key.iter().find_map(|(username, connection_data)| { connection_data - .streams - .iter() - .any(|stream| !stream.preserved && stream.uid == uid && stream.addr == *addr) - .then(|| username.clone()) - }), - None => user_connections - .key_by_addr - .get(addr) - .filter(|reg| !reg.username.is_empty()) - .map(|reg| reg.username.clone()) - .or_else(|| { - user_connections.by_key.iter().find_map(|(username, connection_data)| { - connection_data - .streams - .iter() - .any(|stream| !stream.preserved && stream.addr == *addr) - .then(|| username.clone()) + .stream_request_claims + .get(&uid) + .is_some_and(|claim| { + claim.addr == *addr + && connection_data + .streams + .iter() + .any(|stream| !stream.preserved && stream.uid == claim.stream_uid) }) - }), - }?; + .then(|| username.clone()) + }) + } else { + let mut matching_users = user_connections + .by_key + .iter() + .filter(|(_, connection_data)| { + connection_data.streams.iter().any(|stream| !stream.preserved && stream.addr == *addr) + }) + .map(|(username, _)| username.clone()); + let first = matching_users.next(); + let second = matching_users.next(); + if second.is_some() { + None + } else { + first + } + }; + let Some(username) = username else { + return StreamRequestDetach::NotFound; + }; let mut removed_stream = None; let mut expiry_entry = None; @@ -812,7 +996,52 @@ impl ActiveUserManager { let mut connection_changed = false; let mut promotion = None; if let Some(connection_data) = user_connections.by_key.get_mut(&username) { - let migrated_session_addrs = connection_data.release_addr_from_sessions(addr); + let stream_uid = if let Some(request_uid) = stream_uid { + let Some((display_uid, is_last)) = connection_data.detach_stream_request(request_uid) else { + return StreamRequestDetach::NotFound; + }; + if !is_last { + let has_claim_on_addr = connection_data + .stream_request_claims + .values() + .any(|claim| claim.stream_uid == display_uid && claim.addr == *addr); + let migrated_session_addrs = if has_claim_on_addr { + HashMap::new() + } else { + connection_data.release_addr_from_stream_session(addr, display_uid) + }; + let next_addr = connection_data + .stream_request_claims + .values() + .find(|claim| claim.stream_uid == display_uid) + .map(|claim| claim.addr); + if let Some(stream) = + connection_data.streams.iter_mut().find(|stream| stream.uid == display_uid) + { + stream.addr = next_addr + .or_else(|| { + stream + .session_token + .as_deref() + .and_then(|token| migrated_session_addrs.get(token)) + .copied() + .flatten() + }) + .unwrap_or(stream.addr); + stream.ts = current_time_secs(); + } + let retained = connection_data.streams.iter().find(|stream| stream.uid == display_uid).cloned(); + return retained.map_or(StreamRequestDetach::NotFound, StreamRequestDetach::Retained); + } + Some(display_uid) + } else { + None + }; + let migrated_session_addrs = if let Some(uid) = stream_uid { + connection_data.release_addr_from_stream_session(addr, uid) + } else { + connection_data.release_addr_from_sessions(addr) + }; if let Some(stream_idx) = connection_data.streams.iter().position(|stream| { !stream.preserved && stream_uid.map_or(stream.addr == *addr, |uid| stream.uid == uid && stream.addr == *addr) @@ -853,6 +1082,7 @@ impl ActiveUserManager { removed_stream = Some(connection_data.streams.swap_remove(stream_idx)); } if let Some(removed_stream) = removed_stream.as_ref() { + connection_data.remove_stream_request_claims(removed_stream.uid); if let Some(kind) = connection_data.stream_kinds.remove(&removed_stream.uid) { connection_data.decrement_kind(kind); } @@ -901,8 +1131,9 @@ impl ActiveUserManager { self.enqueue_adaptive_expiry(entry).await; } - if let Some(stream_info) = preserved_update { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + if let Some(stream_info) = preserved_update.as_ref() { + self.event_manager + .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); } if connection_changed { @@ -919,7 +1150,13 @@ impl ActiveUserManager { self.emit_promotion_update(&username, action).await; } - removed_stream + if let Some(stream_info) = removed_stream { + StreamRequestDetach::Removed(stream_info) + } else if let Some(stream_info) = preserved_update { + StreamRequestDetach::Preserved(stream_info) + } else { + StreamRequestDetach::NotFound + } } #[allow(clippy::too_many_lines)] @@ -937,11 +1174,8 @@ impl ActiveUserManager { let registration = user_connections.key_by_addr.remove(addr); let had_registration = registration.is_some(); - let mut disconnected_users = registration - .map(|registration| registration.username) - .filter(|username| !username.is_empty()) - .into_iter() - .collect::>(); + let mut disconnected_users: HashSet = + registration.map(SocketRegistration::into_usernames).unwrap_or_default(); disconnected_users.extend( user_connections .by_key @@ -952,8 +1186,6 @@ impl ActiveUserManager { }) .map(|(username, _)| username.clone()), ); - disconnected_users.sort_unstable(); - disconnected_users.dedup(); let mut removed_streams = Vec::new(); let mut expiry_entries = Vec::new(); @@ -964,6 +1196,7 @@ impl ActiveUserManager { if let Some(connection_data) = user_connections.by_key.get_mut(username) { let previous_connection_count = connection_data.connections; let migrated_session_addrs = connection_data.release_addr_from_sessions(addr); + connection_data.stream_request_claims.retain(|_, claim| claim.addr != *addr); let mut remaining_streams = Vec::with_capacity(connection_data.streams.len()); let mut released_kinds = Vec::new(); let mut removed_session_tokens = HashSet::new(); @@ -1025,6 +1258,10 @@ impl ActiveUserManager { } } connection_data.streams = remaining_streams; + let streams = &connection_data.streams; + connection_data + .stream_request_claims + .retain(|_, claim| streams.iter().any(|stream| stream.uid == claim.stream_uid)); if !preserve_session_streams && !removed_session_tokens.is_empty() { connection_data.sessions.retain(|session| !removed_session_tokens.contains(&session.token)); } @@ -1091,7 +1328,11 @@ impl ActiveUserManager { } } - ReleasedConnection { addr_removed, removed_streams, disconnected_users } + ReleasedConnection { + addr_removed, + removed_streams, + disconnected_users: disconnected_users.into_iter().collect(), + } } pub async fn release_connection(&self, addr: &SocketAddr) -> ReleasedConnection { @@ -1148,7 +1389,7 @@ impl ActiveUserManager { connection_data.granted_grace = false; connection_data.grace_ts = 0; } - return ConnectionAdmission { permission: UserConnectionPermission::Allowed, kind: Some(kind) }; + return ConnectionAdmission::allowed(Some(kind)); } let now = get_current_timestamp(); @@ -1159,7 +1400,7 @@ impl ActiveUserManager { { // Grace timeout, still active, deny connection debug!("User access denied, grace exhausted, too many connections: {username}"); - return ConnectionAdmission { permission: UserConnectionPermission::Exhausted, kind: None }; + return ConnectionAdmission::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, None); } // Grace timeout expired, reset grace counters if effective_connections < connection_data.max_connections { @@ -1169,7 +1410,7 @@ impl ActiveUserManager { } debug!("User access denied, too many connections: {username}"); - ConnectionAdmission { permission: UserConnectionPermission::Exhausted, kind: None } + ConnectionAdmission::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, None) } fn check_connection_admission( @@ -1197,7 +1438,7 @@ impl ActiveUserManager { return self.check_connection_admission(username, connection_data); } } - ConnectionAdmission { permission: UserConnectionPermission::Allowed, kind: Some(ConnectionKind::Normal) } + ConnectionAdmission::allowed(Some(ConnectionKind::Normal)) } pub async fn connection_permission( @@ -1217,18 +1458,12 @@ impl ActiveUserManager { session_token: &str, ) -> ConnectionAdmission { if max_connections == 0 && soft_connections == 0 { - return ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: Some(ConnectionKind::Normal), - }; + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); } let mut connections = self.connections.write().await; let Some(connection_data) = connections.by_key.get_mut(username) else { - return ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: Some(ConnectionKind::Normal), - }; + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); }; connection_data.max_connections = max_connections; connection_data.soft_connections = soft_connections; @@ -1239,10 +1474,9 @@ impl ActiveUserManager { }; if connection_data.sessions[session_index].lifecycle.is_counted() { - return ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), - }; + return ConnectionAdmission::allowed( + connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), + ); } self.check_connection_admission_with_counts( @@ -1261,7 +1495,7 @@ impl ActiveUserManager { ) -> UserConnectionPermission { self.connection_admission_for_session(username, max_connections, soft_connections, session_token) .await - .permission + .permission() } pub async fn refresh_session_connection_kind_for_origin_policy( @@ -1294,15 +1528,15 @@ impl ActiveUserManager { connection_data, connection_data.effective_counts_for_admission(Some(session_token)), ); - if admission.permission == UserConnectionPermission::Allowed { - if let Some(kind) = admission.kind { + if admission.permission() == UserConnectionPermission::Allowed { + if let Some(kind) = admission.kind() { Self::update_session_admission( &mut connection_data.sessions[session_index], - admission.permission, + admission.permission(), Some(kind), ); } - admission.kind + admission.kind() } else { None } @@ -1371,7 +1605,13 @@ impl ActiveUserManager { addr_count == 1 } }) - .map(|s| crate::EvictionCandidate { addr: s.addr, client_ip: s.client_ip.clone(), ts: s.ts }) + .map(|s| crate::EvictionCandidate { + addr: s.addr, + client_ip: s.client_ip.clone(), + virtual_id: shared::model::VirtualId::new(s.channel.virtual_id), + ts: s.ts, + uid: s.uid, + }) .collect(); candidates } @@ -1423,21 +1663,37 @@ impl ActiveUserManager { .fold((0usize, 0usize), |(user_count, conn_count), effective| (user_count + 1, conn_count + effective)) } + pub async fn playback_resource_counts(&self) -> (usize, usize) { + self.gc(); + let user_connections = self.connections.read().await; + user_connections.by_key.values().fold((0, 0), |(requests, sessions), connection| { + (requests + connection.stream_request_claims.len(), sessions + connection.sessions.len()) + }) + } + pub async fn update_stream_detail( &self, addr: &SocketAddr, video_type: CustomVideoStreamType, ) -> Option { - let mut user_connections = self.connections.write().await; - let username = { - match user_connections.key_by_addr.get(addr) { - Some(registration) => registration.username.clone(), - None => return None, + let uid = { + let connections = self.connections.read().await; + let mut streams = + connections.by_key.values().flat_map(|data| &data.streams).filter(|stream| &stream.addr == addr); + let uid = streams.next()?.uid; + if streams.next().is_some() { + return None; } + uid }; - if let Some(connection_data) = user_connections.by_key.get_mut(&username) { + self.update_stream_detail_by_uid(uid, video_type).await + } + + pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) -> Option { + let mut user_connections = self.connections.write().await; + for connection_data in user_connections.by_key.values_mut() { for stream in &mut connection_data.streams { - if &stream.addr == addr { + if stream.uid == uid { // IMPORTANT: `resolve_disconnect_reason` in connection_manager.rs parses // `channel.title` back via `CustomVideoStreamType::from_str` to determine QoS // disconnect reasons. If these values change, update that function too. @@ -1483,14 +1739,9 @@ impl ActiveUserManager { let mut user_connections = self.connections.write().await; let now = current_time_secs(); - if let Some(registration) = user_connections.key_by_addr.get_mut(&fingerprint.addr) { - registration.username = username.to_string(); - registration.ts = now; - } else { - user_connections - .key_by_addr - .insert(fingerprint.addr, SocketRegistration { username: username.to_string(), ts: now }); - } + let registration = + user_connections.key_by_addr.entry(fingerprint.addr).or_insert_with(SocketRegistration::anonymous); + registration.add_user(username, now); let tracked_socket_count = user_connections.key_by_addr.len(); let connection_data = user_connections @@ -1521,12 +1772,15 @@ impl ActiveUserManager { let existing_stream_info = connection_data .streams .iter() - .position(|stream_info| match session_token { - Some(token) => { - stream_info.session_token.as_deref() == Some(token) - && Self::should_reuse_stream_for_session(stream_info, stream_channel) - } - None => stream_info.addr == fingerprint.addr && stream_info.session_token.is_none(), + .position(|stream_info| { + !stream_channel.shared + && match session_token { + Some(token) => { + stream_info.session_token.as_deref() == Some(token) + && Self::should_reuse_stream_for_session(stream_info, stream_channel) + } + None => stream_info.uid == uid && stream_info.session_token.is_none(), + } }) .map(|stream_idx| { let session_started_at = session_token.and_then(|token| { @@ -1575,6 +1829,7 @@ impl ActiveUserManager { }); let (stream_info, divergence_snapshot) = if let Some((stream_info, was_preserved)) = existing_stream_info { let effective_connection_kind = reserved_session_kind.unwrap_or(connection_kind); + connection_data.attach_stream_request(uid, stream_info.uid, fingerprint.addr); if was_preserved { connection_data.increment_kind(effective_connection_kind); } @@ -1614,6 +1869,7 @@ impl ActiveUserManager { connection_data.increment_kind(effective_connection_kind); } connection_data.streams.push(stream_info.clone()); + connection_data.attach_stream_request(uid, stream_info.uid, fingerprint.addr); connection_data.stream_kinds.insert(stream_info.uid, effective_connection_kind); connection_data.stream_normal_priorities.insert(stream_info.uid, priority); if let Some(token) = session_token { @@ -1868,18 +2124,12 @@ impl ActiveUserManager { session_token: &str, ) -> ConnectionAdmission { if max_connections == 0 && soft_connections == 0 { - return ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: Some(ConnectionKind::Normal), - }; + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); } let mut connections = self.connections.write().await; let Some(connection_data) = connections.by_key.get_mut(username) else { - return ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: Some(ConnectionKind::Normal), - }; + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); }; connection_data.max_connections = max_connections; connection_data.soft_connections = soft_connections; @@ -1896,10 +2146,9 @@ impl ActiveUserManager { if connection_data.sessions[session_index].lifecycle.is_counted() || Self::session_has_stream(connection_data, session_token) { - return ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), - }; + return ConnectionAdmission::allowed( + connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), + ); } // Uncounted session with no stream row: run normal admission. @@ -1910,9 +2159,9 @@ impl ActiveUserManager { connection_data, connection_data.effective_counts_for_admission(Some(session_token)), ); - if admission.permission == UserConnectionPermission::Allowed { + if admission.permission() == UserConnectionPermission::Allowed { let session = &mut connection_data.sessions[session_index]; - Self::update_session_admission(session, admission.permission, admission.kind); + Self::update_session_admission(session, admission.permission(), admission.kind()); } admission } @@ -2218,7 +2467,7 @@ impl ActiveUserManager { session.ts = now; Self::bump_session_transition_version(session); for stream in &mut connection_data.streams { - if stream.addr == previous_addr { + if stream.addr == previous_addr && stream.session_token.as_deref() == Some(token) { stream.addr = *addr; stream.ts = now; } @@ -2232,19 +2481,14 @@ impl ActiveUserManager { }; if let Some((previous_addr, prune_previous_registration)) = update_result { - if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { - registration.ts = now; - registration.username = username.to_string(); - } else { - user_connections - .key_by_addr - .insert(*addr, SocketRegistration { username: username.to_string(), ts: now }); - } + let registration = + user_connections.key_by_addr.entry(*addr).or_insert_with(SocketRegistration::anonymous); + registration.add_user(username, now); if prune_previous_registration { let can_remove_previous = user_connections .key_by_addr - .get(&previous_addr) - .is_some_and(|registration| registration.username == username); + .get_mut(&previous_addr) + .is_some_and(|registration| registration.remove_user(username)); if can_remove_previous { user_connections.key_by_addr.remove(&previous_addr); } @@ -2282,33 +2526,20 @@ impl ActiveUserManager { } else { false }; - if !cleared { - let can_remove_registration = !connection_data.has_session_addr(addr) - && !connection_data.streams.iter().any(|stream| stream.addr == *addr); - if can_remove_registration { - let can_remove = user_connections - .key_by_addr - .get(addr) - .is_some_and(|registration| registration.username.is_empty() || registration.username == username); - if can_remove { - user_connections.key_by_addr.remove(addr); - } - } - return; - } - let can_remove_registration = !connection_data.has_session_addr(addr) && !connection_data.streams.iter().any(|stream| stream.addr == *addr); if can_remove_registration { let can_remove = user_connections .key_by_addr - .get(addr) - .is_some_and(|registration| registration.username.is_empty() || registration.username == username); + .get_mut(addr) + .is_some_and(|registration| registration.remove_user(username)); if can_remove { user_connections.key_by_addr.remove(addr); } - } else if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { - registration.ts = now; + } else if cleared { + if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { + registration.ts = now; + } } } @@ -2501,6 +2732,7 @@ impl ActiveUserManager { connection_data.decrement_kind(kind); } connection_data.stream_normal_priorities.remove(&connection_data.streams[stream_idx].uid); + connection_data.remove_stream_request_claims(connection_data.streams[stream_idx].uid); connection_data.streams.swap_remove(stream_idx); removed_count += 1; } @@ -2735,32 +2967,49 @@ impl ActiveUserManager { } pub async fn socket_expiry_deadline(&self, addr: &SocketAddr) -> Option { - let ttl_secs = self.active_socket_ttl_secs(); let connections = self.connections.read().await; let registration = connections.key_by_addr.get(addr)?; - if registration.username.is_empty() { + if registration.is_empty() { return None; } + let ttl_secs = self.effective_socket_ttl_secs(&connections, addr); Some(registration.ts.saturating_add(ttl_secs)) } + /// Direct VOD/series bodies may stop reading while the player drains its buffer. + /// Their socket expiry must not fire before the direct-body idle timeout, or a + /// buffering player is disconnected mid-playback even though the stream is alive. + fn effective_socket_ttl_secs(&self, connections: &UserConnections, addr: &SocketAddr) -> u64 { + let base_ttl = self.active_socket_ttl_secs(); + if Self::addr_has_direct_body_stream(connections, addr) { + base_ttl.max(DIRECT_BODY_IDLE_TIMEOUT_SECS) + } else { + base_ttl + } + } + + fn addr_has_direct_body_stream(connections: &UserConnections, addr: &SocketAddr) -> bool { + let Some(registration) = connections.key_by_addr.get(addr) else { + return false; + }; + registration.usernames.iter().any(|username| { + connections.by_key.get(username).is_some_and(|connection_data| { + connection_data + .streams + .iter() + .any(|stream| stream.addr == *addr && uses_direct_body_idle_timeout(&stream.channel)) + }) + }) + } + pub async fn touch_socket_activity(&self, addr: &SocketAddr) { let now = current_time_secs(); let mut user_connections = self.connections.write().await; - let Some(username) = user_connections.key_by_addr.get_mut(addr).and_then(|registration| { - if registration.username.is_empty() { - None - } else { - registration.ts = now; - Some(registration.username.clone()) - } - }) else { - return; - }; - - if let Some(connection_data) = user_connections.by_key.get_mut(&username) { - connection_data.ts = now; + // Transport activity keeps the transport alive, not every playback that + // has previously used it. Authenticated HTTP activity updates its own user. + if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { + registration.ts = now; } } @@ -2769,8 +3018,7 @@ impl ActiveUserManager { let mut user_connections = self.connections.write().await; let registration = user_connections.key_by_addr.entry(*addr).or_insert_with(SocketRegistration::anonymous); - registration.username = username.to_string(); - registration.ts = now; + registration.add_user(username, now); let Some(connection_data) = user_connections.by_key.get_mut(username) else { return; @@ -2852,11 +3100,19 @@ impl ActiveUserManager { /// Allocates the next process-local, globally unique User-Agent stream index. pub fn next_user_agent_stream_index(&self) -> u64 { - self.next_user_agent_stream_index - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { - Some(if current == 0 || current == u64::MAX { 1 } else { current + 1 }) - }) - .map_or(1, |val| if val == 0 { 1 } else { val }) + let mut current = self.next_user_agent_stream_index.load(Ordering::Relaxed); + loop { + let next = if current == 0 || current == u64::MAX { 1 } else { current + 1 }; + match self.next_user_agent_stream_index.compare_exchange_weak( + current, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(previous) => return if previous == 0 { 1 } else { previous }, + Err(observed) => current = observed, + } + } } /// Persists a previously allocated index without replacing an existing session identity. @@ -2900,9 +3156,9 @@ impl ActiveUserManager { && !matches!(connection_data.sessions[session_index].lifecycle, PlaybackLifecycle::PendingProvider { .. }) { let admission = self.check_connection_admission(username, connection_data); - connection_data.sessions[session_index].permission = admission.permission; - if admission.kind.is_some() { - connection_data.sessions[session_index].connection_kind = admission.kind; + connection_data.sessions[session_index].permission = admission.permission(); + if admission.kind().is_some() { + connection_data.sessions[session_index].connection_kind = admission.kind(); } } @@ -2987,9 +3243,8 @@ impl ActiveUserManager { pub async fn recently_evicted_session_protected_addr(&self, session_token: &str) -> Option { let connections = self.connections.read().await; - let now = current_time_secs(); let protection = connections.recently_evicted_sessions.get(session_token)?; - if protection.expires_at > now { + if protection.expires_at > Instant::now() { return Some(protection.protected_addr); } @@ -2999,7 +3254,7 @@ impl ActiveUserManager { connections .key_by_addr .get(&protection.protected_addr) - .filter(|registration| registration.username == username) + .filter(|registration| registration.usernames.contains(username)) .map(|_| protection.protected_addr) } @@ -3010,18 +3265,13 @@ impl ActiveUserManager { virtual_id: VirtualId, ) -> Option { let connections = self.connections.read().await; - let now = current_time_secs(); let key = create_socket_reentry_guard_key(username, client_ip, virtual_id); let protection = connections.recent_socket_reentry_guards.get(&key)?; - if protection.expires_at > now { - return Some(protection.protected_addr); + if protection.expires_at > Instant::now() { + Some(protection.protected_addr) + } else { + None } - - connections - .key_by_addr - .get(&protection.protected_addr) - .filter(|registration| registration.username == username) - .map(|_| protection.protected_addr) } pub async fn block_user_for_stream(&self, addr: &SocketAddr, virtual_id: VirtualId, blocked_secs: u64) { @@ -3030,62 +3280,82 @@ impl ActiveUserManager { let mut connections = self.connections.write().await; let now = current_time_secs(); connections.kicked.retain(|_, (expires_at, _)| *expires_at > now); - if let Some(username) = connections + let candidate_usernames: Vec = connections .key_by_addr .get(addr) - .map(|registration| registration.username.clone()) - .filter(|username| !username.is_empty()) - { - let expires_at = now + block_for_secs; - connections.kicked.insert(username, (expires_at, virtual_id)); + .map(|reg| reg.usernames.iter().cloned().collect()) + .unwrap_or_default(); + let target_vid = virtual_id.get(); + let expires_at = now + block_for_secs; + for username in candidate_usernames { + // Only block users who are actually watching the target channel + // at this socket address. Behind reverse proxies / NAT, multiple + // distinct users share the same SocketAddr; blocking all of them + // would be a multi-tenant isolation violation. + let is_watching_target = connections.by_key.get(&username).is_some_and(|data| { + data.streams.iter().any(|s| s.addr == *addr && s.channel.virtual_id == target_vid) + }); + if is_watching_target { + connections.kicked.insert(username, (expires_at, virtual_id)); + } } } } + pub async fn block_user_for_stream_uid(&self, uid: u32, virtual_id: VirtualId, blocked_secs: u64) { + if blocked_secs == 0 { + return; + } + let mut connections = self.connections.write().await; + let username = connections + .by_key + .iter() + .find_map(|(username, data)| data.streams.iter().any(|stream| stream.uid == uid).then(|| username.clone())); + if let Some(username) = username { + connections.kicked.insert(username, (current_time_secs() + blocked_secs.min(86_400), virtual_id)); + } + } + pub async fn mark_recent_eviction_guard_for_addr( &self, addr: &SocketAddr, protected_addr: SocketAddr, - ttl_secs: u64, + ttl: Duration, ) { - if ttl_secs == 0 { + if ttl.is_zero() { return; } let mut connections = self.connections.write().await; - let now = current_time_secs(); + let now = Instant::now(); connections.recently_evicted_sessions.retain(|_, protection| protection.expires_at > now); connections.recent_socket_reentry_guards.retain(|_, protection| protection.expires_at > now); - let Some(username) = connections - .key_by_addr - .get(addr) - .map(|registration| registration.username.clone()) - .filter(|username| !username.is_empty()) - else { + let usernames = connections.key_by_addr.get(addr).map(SocketRegistration::all_usernames).unwrap_or_default(); + if usernames.is_empty() { return; - }; + } - let Some(connection_data) = connections.by_key.get(&username) else { - return; - }; - - let protection = RecentWinnerProtection { protected_addr, expires_at: now + ttl_secs }; + let protection = RecentWinnerProtection { protected_addr, expires_at: now + ttl }; let mut session_tokens = Vec::new(); let mut socket_guard_keys = Vec::new(); - for stream in connection_data.streams.iter().filter(|stream| stream.addr == *addr) { - if uses_session_reentry_guard(stream) && stream.session_token.is_some() { - let Some(session_token) = stream.session_token.clone() else { - continue; - }; - session_tokens.push(session_token); - } else { - socket_guard_keys.push(create_socket_reentry_guard_key( - &username, - &stream.client_ip, - shared::model::VirtualId::new(stream.channel.virtual_id), - )); + for username in &usernames { + if let Some(connection_data) = connections.by_key.get(username) { + for stream in connection_data.streams.iter().filter(|stream| stream.addr == *addr) { + if uses_session_reentry_guard(stream) && stream.session_token.is_some() { + let Some(session_token) = stream.session_token.clone() else { + continue; + }; + session_tokens.push(session_token); + } else { + socket_guard_keys.push(create_socket_reentry_guard_key( + username, + &stream.client_ip, + shared::model::VirtualId::new(stream.channel.virtual_id), + )); + } + } } } @@ -3098,7 +3368,17 @@ impl ActiveUserManager { } pub async fn get_username_for_addr(&self, addr: &SocketAddr) -> Option { - self.connections.read().await.key_by_addr.get(addr).map(|registration| registration.username.clone()) + self.connections + .read() + .await + .key_by_addr + .get(addr) + .and_then(SocketRegistration::primary_username) + .map(str::to_string) + } + + pub async fn get_usernames_for_addr(&self, addr: &SocketAddr) -> Vec { + self.connections.read().await.key_by_addr.get(addr).map(SocketRegistration::all_usernames).unwrap_or_default() } fn should_preserve_session_stream(stream: &StreamInfo) -> bool { @@ -3311,6 +3591,7 @@ impl ActiveUserManager { removed_addrs.push(addr); } let removed_stream = connection_data.streams.swap_remove(stream_idx); + connection_data.remove_stream_request_claims(removed_stream.uid); if let Some(kind) = connection_data.stream_kinds.remove(&removed_stream.uid) { connection_data.decrement_kind(kind); } @@ -3400,8 +3681,13 @@ impl ActiveUserManager { { if let Ok(mut user_connections) = self.connections.try_write() { user_connections.kicked.retain(|_, (expires_at, _)| *expires_at > now); - user_connections.recently_evicted_sessions.retain(|_, protection| protection.expires_at > now); - user_connections.recent_socket_reentry_guards.retain(|_, protection| protection.expires_at > now); + let now_instant = Instant::now(); + user_connections + .recently_evicted_sessions + .retain(|_, protection| protection.expires_at > now_instant); + user_connections + .recent_socket_reentry_guards + .retain(|_, protection| protection.expires_at > now_instant); for connection_data in user_connections.by_key.values_mut() { Self::release_expired_session_reservations(connection_data, now); connection_data.sessions.retain(|s| now.saturating_sub(s.ts) < USER_CON_TTL); @@ -3409,8 +3695,14 @@ impl ActiveUserManager { user_connections.by_key.retain(|_k, v| { v.connections > 0 || !v.streams.is_empty() || now.saturating_sub(v.ts) < USER_CON_TTL }); - user_connections.key_by_addr.retain(|_, registration| { - !(registration.username.is_empty() && now.saturating_sub(registration.ts) >= ANON_SOCKET_TTL) + let UserConnections { ref by_key, ref mut key_by_addr, .. } = *user_connections; + key_by_addr.retain(|addr, registration| { + registration.usernames.retain(|u| { + by_key + .get(u) + .is_some_and(|d| d.has_session_addr(addr) || d.streams.iter().any(|s| s.addr == *addr)) + }); + !(registration.is_empty() && now.saturating_sub(registration.ts) >= ANON_SOCKET_TTL) }); } else { // Lock contention: release the GC claim so a subsequent caller can retry immediately. diff --git a/backend/session/src/active_user_manager/tests.rs b/backend/session/src/active_user_manager/tests.rs index 17473b82c..908532402 100644 --- a/backend/session/src/active_user_manager/tests.rs +++ b/backend/session/src/active_user_manager/tests.rs @@ -2066,17 +2066,17 @@ async fn recently_evicted_session_guard_survives_ttl_while_protected_addr_is_sti }) .await; - manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, 1).await; + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(1)).await; { let mut connections = manager.connections.write().await; if let Some(registration) = connections.key_by_addr.get_mut(&protected_addr) { - registration.username = user.username.clone(); + registration.add_user(&user.username, current_time_secs()); } let protection = connections .recently_evicted_sessions .get_mut("tok-guard-session") .expect("recent eviction guard should exist"); - protection.expires_at = current_time_secs().saturating_sub(1); + protection.expires_at = Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now); } assert_eq!(manager.recently_evicted_session_protected_addr("tok-guard-session").await, Some(protected_addr)); @@ -2133,7 +2133,7 @@ async fn recently_evicted_vod_uses_session_reentry_guard() { }) .await; - manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, 10).await; + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; assert_eq!(manager.recently_evicted_session_protected_addr("tok-guard-vod").await, Some(protected_addr)); let connections = manager.connections.read().await; @@ -2177,7 +2177,7 @@ async fn provider_affine_stream_without_session_token_uses_socket_reentry_fallba }) .await; - manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, 10).await; + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; assert_eq!( manager @@ -2191,6 +2191,56 @@ async fn provider_affine_stream_without_session_token_uses_socket_reentry_fallba ); } +#[tokio::test] +async fn socket_reentry_guard_expires_at_ttl() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let evicted_addr: SocketAddr = "127.0.0.1:55117".parse().unwrap(); + let protected_addr: SocketAddr = "127.0.0.1:55118".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-socket-ttl".to_string(), "127.0.0.1".to_string(), evicted_addr); + let channel = test_channel(2021); + + manager.add_connection(&evicted_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 21, + meter_uid: 0, + username: "socket-ttl-user", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await; + + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; + let virtual_id = shared::model::VirtualId::new(channel.virtual_id); + assert_eq!( + manager.recent_socket_reentry_protected_addr("socket-ttl-user", "127.0.0.1", virtual_id).await, + Some(protected_addr) + ); + + // Expire only the guard while the protected address stays registered: the TTL is + // authoritative and must not fall back to the still-present socket registration. + { + let mut connections = manager.connections.write().await; + let key = create_socket_reentry_guard_key("socket-ttl-user", "127.0.0.1", virtual_id); + let protection = + connections.recent_socket_reentry_guards.get_mut(&key).expect("socket reentry guard should exist"); + protection.expires_at = Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now); + } + assert_eq!(manager.recent_socket_reentry_protected_addr("socket-ttl-user", "127.0.0.1", virtual_id).await, None); +} + #[tokio::test] async fn test_reused_logical_stream_refreshes_normal_priority() { let config = Config::default(); @@ -2433,7 +2483,7 @@ async fn unlimited_user_can_open_same_and_different_live_streams_from_same_ip() } #[tokio::test] -async fn release_stream_by_uid_removes_only_matching_stream_on_shared_addr() { +async fn affine_request_claims_keep_playback_until_oldest_body_finishes() { let config = Config::default(); let geoip = Arc::new(ArcSwapOption::::default()); let event_manager = Arc::new(EventManager::new()); @@ -2458,19 +2508,140 @@ async fn release_stream_by_uid_removes_only_matching_stream_on_shared_addr() { soft_priority: 0, fingerprint: &fingerprint, provider: "provider-a".intern(), - stream_channel: &test_channel(3004), + stream_channel: &test_adaptive_channel(3004), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-live-shared-addr"), }) .await; } - let removed = manager.release_stream_by_uid(&addr, 42).await; - assert!(removed.as_ref().is_some_and(|stream| stream.uid == 42)); + assert_eq!(manager.active_streams().await.len(), 1); + assert!(manager.release_stream_by_uid(&addr, 42).await.is_none()); let streams = manager.active_streams().await; assert_eq!(streams.len(), 1); assert_eq!(streams[0].uid, 41); + + let removed = manager.release_stream_by_uid(&addr, 41).await; + assert!(removed.as_ref().is_some_and(|stream| stream.uid == 41)); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn affine_request_claims_keep_playback_until_newest_body_finishes() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + let addr: SocketAddr = "127.0.0.1:55037".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-affine-cleanup-order".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + for uid in [45, 46] { + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: 0, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3005), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-affine-cleanup-order"), + }) + .await + .expect("affine request should register"); + } + + assert!(manager.release_stream_by_uid(&addr, 45).await.is_none()); + assert_eq!(manager.active_users_and_connections().await, (1, 1)); + assert_eq!(manager.active_streams().await.len(), 1); + + let removed = manager.release_stream_by_uid(&addr, 46).await; + assert!(removed.as_ref().is_some_and(|stream| stream.uid == 45)); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + assert!(manager.active_streams().await.is_empty()); +} + +/// PR1 regression: two bodies with distinct request UIDs that map to the same +/// display stream must both clean up correctly. The current body path passes the +/// *display* UID (`stream_info.uid`) to cleanup, not the request UID. This test +/// asserts that the cleanup is request-affine β€” releasing the display UID twice +/// must drain both claims, and the second release must find the stream. +#[tokio::test] +async fn pr1_two_bodies_same_display_uid_both_cleanups_must_succeed() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55035".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-pr1-display-vs-request".to_string(), "127.0.0.1".to_string(), addr); + manager.add_connection(&addr).await; + + // Request 41 creates the display stream. + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 41, + meter_uid: 301, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3006), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-pr1"), + }) + .await + .expect("first body should register"); + + // Request 42 reuses the same playback (same session token, same channel). + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 42, + meter_uid: 302, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3006), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-pr1"), + }) + .await + .expect("second body should register"); + + // Both return the same display UID. + assert_eq!(first.uid, second.uid, "display UID must be stable across requests"); + let display_uid = first.uid; + + // Fixed behavior: the body passes its own request UID to cleanup, not the display UID. + // First body (request 41) finishes β†’ should NOT remove the stream (request 42 still active). + let removed_first = manager.release_stream_by_uid(&addr, 41).await; + assert!(removed_first.is_none(), "first body cleanup must keep stream alive for second body"); + + assert_eq!(manager.active_streams().await.len(), 1, "stream must survive first body cleanup"); + + // Second body (request 42) finishes β†’ should remove the stream (no more claims). + let removed_second = manager.release_stream_by_uid(&addr, 42).await; + assert!(removed_second.is_some(), "second body cleanup must find and remove the stream"); + assert_eq!(removed_second.unwrap().uid, display_uid); + + assert!(manager.active_streams().await.is_empty(), "no streams left after all bodies finish"); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); } #[tokio::test] @@ -2621,7 +2792,7 @@ async fn connection_counts_are_broadcast_when_active_user_logging_is_disabled() manager .update_connection(ActiveUserConnectionParams { - uid: 49, + uid: 48, meter_uid: 0, username: "event-user", max_connections: 1, @@ -3890,7 +4061,7 @@ async fn clear_unbound_session_addr_prunes_touch_only_manifest_addr() { } #[tokio::test] -async fn socket_expiry_deadline_does_not_refresh_active_vod_streams_without_activity() { +async fn socket_expiry_deadline_extends_direct_body_streams_to_the_idle_timeout() { let config = Config::default(); let geoip = Arc::new(ArcSwapOption::::default()); let event_manager = Arc::new(EventManager::new()); @@ -3941,10 +4112,13 @@ async fn socket_expiry_deadline_does_not_refresh_active_vod_streams_without_acti .await .expect("vod stream should be created"); + // The player stops reading while draining its buffer. The pause already exceeds the + // short HLS session TTL but stays well within the direct-body idle timeout, so the + // socket must not be treated as expired yet. let previous_registration_ts = { let mut connections = manager.connections.write().await; let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); - registration.ts = registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); + registration.ts = registration.ts.saturating_sub(default_hls_session_ttl_secs() + 5); registration.ts }; @@ -3957,10 +4131,77 @@ async fn socket_expiry_deadline_does_not_refresh_active_vod_streams_without_acti }; assert_eq!(unchanged_registration_ts, previous_registration_ts); + assert!( + deadline > current_time_secs(), + "a buffering direct-body stream must not be treated as expired after only the HLS session TTL" + ); assert_eq!( deadline, - previous_registration_ts.saturating_add(manager.active_socket_ttl_secs()), - "deadline checks must not refresh VOD sockets without real body activity" + previous_registration_ts.saturating_add(DIRECT_BODY_IDLE_TIMEOUT_SECS), + "direct-body sockets must use the direct-body idle timeout as their expiry allowance" + ); +} + +#[tokio::test] +async fn socket_expiry_deadline_keeps_hls_session_ttl_for_adaptive_streams() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55043".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-hls".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-hls-expiry".to_string(); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-expiry", + virtual_id: 8891, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + manager + .update_connection(ActiveUserConnectionParams { + uid: 605, + meter_uid: 705, + username: "user-hls-expiry", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(8891), + user_agent: Cow::Borrowed("player/1.0"), + session_token: Some("tok-hls-expiry"), + }) + .await + .expect("hls stream should be created"); + + let registration_ts = { + let mut connections = manager.connections.write().await; + let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); + registration.ts = registration.ts.saturating_sub(default_hls_session_ttl_secs() + 5); + registration.ts + }; + + let deadline = manager.socket_expiry_deadline(&addr).await.expect("HLS streams should stay scheduled for expiry"); + + assert_eq!( + deadline, + registration_ts.saturating_add(manager.active_socket_ttl_secs()), + "adaptive sockets keep the short HLS session TTL" ); } @@ -4389,9 +4630,9 @@ async fn session_activation_keeps_first_hls_slot_uncommitted_before_stream_regis .connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-second") .await; - assert_eq!(first_admission.permission, UserConnectionPermission::Allowed); - assert_eq!(first_admission.kind, Some(ConnectionKind::Normal)); - assert_eq!(second_admission.permission, UserConnectionPermission::Allowed); + assert_eq!(first_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(first_admission.kind(), Some(ConnectionKind::Normal)); + assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); let connections = manager.connections.read().await; let connection_data = connections.by_key.get(&user.username).expect("user connection data"); @@ -4466,8 +4707,8 @@ async fn binding_reserved_sessions_keeps_hard_and_soft_counts_stable() { "tok-normal", ) .await; - assert_eq!(first_admission.permission, UserConnectionPermission::Allowed); - assert_eq!(first_admission.kind, Some(ConnectionKind::Normal)); + assert_eq!(first_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(first_admission.kind(), Some(ConnectionKind::Normal)); manager .update_connection(ActiveUserConnectionParams { @@ -4496,8 +4737,8 @@ async fn binding_reserved_sessions_keeps_hard_and_soft_counts_stable() { "tok-soft", ) .await; - assert_eq!(second_admission.permission, UserConnectionPermission::Allowed); - assert_eq!(second_admission.kind, Some(ConnectionKind::Soft)); + assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(second_admission.kind(), Some(ConnectionKind::Soft)); manager .update_connection(ActiveUserConnectionParams { @@ -4591,8 +4832,8 @@ async fn origin_policy_refresh_promotes_counted_soft_session_when_hard_slot_is_a "tok-normal", ) .await; - assert_eq!(normal_admission.permission, UserConnectionPermission::Allowed); - assert_eq!(normal_admission.kind, Some(ConnectionKind::Normal)); + assert_eq!(normal_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(normal_admission.kind(), Some(ConnectionKind::Normal)); manager .update_connection(ActiveUserConnectionParams { uid: 411, @@ -4620,8 +4861,8 @@ async fn origin_policy_refresh_promotes_counted_soft_session_when_hard_slot_is_a "tok-soft", ) .await; - assert_eq!(soft_admission.permission, UserConnectionPermission::Allowed); - assert_eq!(soft_admission.kind, Some(ConnectionKind::Soft)); + assert_eq!(soft_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(soft_admission.kind(), Some(ConnectionKind::Soft)); manager .update_connection(ActiveUserConnectionParams { uid: 412, @@ -4795,7 +5036,7 @@ async fn release_unbound_session_reservation_frees_reserved_slot() { let admission = manager .connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-release") .await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); manager.release_unbound_session_reservation(&user.username, "tok-release", None, false).await; @@ -4828,7 +5069,7 @@ async fn preserved_reactivation_admission_does_not_create_ownerless_counted_slot let virtual_admission = manager.connection_admission(&user.username, user.max_connections, user.soft_connections).await; - assert_eq!(virtual_admission.permission, UserConnectionPermission::Exhausted); + assert_eq!(virtual_admission.permission(), UserConnectionPermission::Exhausted); let admission = manager .connection_admission_for_session_activation( @@ -4839,8 +5080,8 @@ async fn preserved_reactivation_admission_does_not_create_ownerless_counted_slot ) .await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); - assert_eq!(admission.kind, Some(ConnectionKind::Normal)); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(ConnectionKind::Normal)); let connections = manager.connections.read().await; let connection_data = connections.by_key.get(&user.username).expect("user connection data"); assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); @@ -4869,7 +5110,7 @@ async fn preserved_reactivation_admission_then_kicked_release_removes_state_with session_token, ) .await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); { let connections = manager.connections.read().await; let connection_data = connections.by_key.get(&user.username).expect("user connection data"); @@ -4917,7 +5158,7 @@ async fn preserved_reactivation_admission_then_lease_idle_cleanup_leaves_counter session_token, ) .await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); { let connections = manager.connections.read().await; let connection_data = connections.by_key.get(&user.username).expect("user connection data"); @@ -4966,7 +5207,7 @@ async fn repeated_preserved_reactivation_cleanup_does_not_accumulate_connections session_token, ) .await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); let counter_changed = manager.release_session_streams_and_counted_reservation(&user.username, session_token).await; @@ -5004,7 +5245,7 @@ async fn dashboard_counts_only_real_slots_during_preserved_reactivation_admissio session_token, ) .await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(manager.active_users_and_connections().await, (0, 0)); let connections = manager.connections.read().await; @@ -5078,8 +5319,8 @@ async fn preserved_soft_reactivation_and_cleanup_leave_normal_slot_unchanged() { session_token, ) .await; - assert_eq!(admission.permission, UserConnectionPermission::Allowed); - assert_eq!(admission.kind, Some(ConnectionKind::Soft)); + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(ConnectionKind::Soft)); { let connections = manager.connections.read().await; @@ -5394,7 +5635,7 @@ async fn connection_admission_treats_preserved_stream_as_reserved_capacity() { let admission = manager.connection_admission(username, user.max_connections, 0).await; assert_eq!( - admission.permission, + admission.permission(), UserConnectionPermission::Exhausted, "a preserved stream should still reserve capacity against unrelated playback admissions" ); @@ -5472,7 +5713,7 @@ async fn connection_admission_for_session_evaluates_admission_for_uncounted_sess // freeing a slot for the uncounted session to reactivate let admission = manager.connection_admission_for_session(username, 1, 0, "tok-uncounted").await; assert_eq!( - admission.permission, + admission.permission(), UserConnectionPermission::Exhausted, "uncounted session should not bypass admission when user is at limit; \ bug: session exists -> Allowed -> strategy evaluation skipped" @@ -5712,3 +5953,140 @@ async fn divergence_log_rate_limited_within_cooldown_window() { assert_eq!(entry.count_since_last_log, 2); assert_eq!(entry.last_logged, first_logged); } + +#[test] +fn socket_registration_primary_username_is_deterministic() { + let mut reg = SocketRegistration::anonymous(); + reg.add_user("user_z", 100); + reg.add_user("user_a", 100); + reg.add_user("user_m", 100); + assert_eq!(reg.primary_username(), Some("user_a")); +} + +#[tokio::test] +async fn release_stream_without_uid_rejects_ambiguous_multiple_users_on_same_addr() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "192.168.1.100:12345".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); + + manager.add_connection(&addr).await; + + let stream_a = manager + .update_connection(ActiveUserConnectionParams { + uid: 1, + meter_uid: 0, + username: "user_a", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(1001), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_a"); + + let stream_b = manager + .update_connection(ActiveUserConnectionParams { + uid: 2, + meter_uid: 0, + username: "user_b", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-b".intern(), + stream_channel: &test_channel(1002), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_b"); + + // Releasing without stream_uid when multiple users are on the same addr must return None (ambiguous) + let released = manager.release_stream(&addr).await; + assert!( + released.is_none(), + "releasing without uid must not arbitrarily pick a user when multiple are active on the same addr" + ); + + // But releasing with explicit stream_uid works cleanly: + let released_a = manager.release_stream_by_uid(&addr, stream_a.uid).await; + assert!(released_a.is_some(), "releasing with stream_uid succeeds"); + + // Now only user_b remains on addr. Releasing without stream_uid should now succeed: + let released_b = manager.release_stream(&addr).await; + assert!(released_b.is_some(), "releasing without stream_uid succeeds when only a single user remains on addr"); + assert_eq!(released_b.unwrap().uid, stream_b.uid); +} + +#[tokio::test] +async fn arm_eviction_protection_protects_all_users_on_addr() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "192.168.1.100:12345".parse().unwrap(); + let protected_addr: SocketAddr = "192.168.1.101:54321".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); + + manager.add_connection(&addr).await; + + manager + .update_connection(ActiveUserConnectionParams { + uid: 1, + meter_uid: 0, + username: "user_a", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(2001), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_a"); + + manager + .update_connection(ActiveUserConnectionParams { + uid: 2, + meter_uid: 0, + username: "user_b", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-b".intern(), + stream_channel: &test_channel(2002), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_b"); + + manager.mark_recent_eviction_guard_for_addr(&addr, protected_addr, Duration::from_secs(60)).await; + + let connections = manager.connections.read().await; + let key_a = create_socket_reentry_guard_key("user_a", "192.168.1.100", shared::model::VirtualId::new(2001)); + let key_b = create_socket_reentry_guard_key("user_b", "192.168.1.100", shared::model::VirtualId::new(2002)); + + assert!(connections.recent_socket_reentry_guards.contains_key(&key_a), "user_a guard must be armed"); + assert!(connections.recent_socket_reentry_guards.contains_key(&key_b), "user_b guard must be armed"); +} diff --git a/backend/session/src/admission.rs b/backend/session/src/admission.rs index 618450649..235b69b1d 100644 --- a/backend/session/src/admission.rs +++ b/backend/session/src/admission.rs @@ -32,9 +32,21 @@ pub struct AdmissionCtx { pub connection_manager: Arc, } -/// How long an eviction is remembered so the evicted client's immediate retry -/// does not evict its own replacement. -pub const RECENT_EVICTION_REENTRY_TTL_SECS: u64 = 3; +/// Default duration an eviction is remembered if not configured. +pub const DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS: u64 = shared::defaults::DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS; + +/// Reentry cooldown from a resolved stream config, falling back to the default when +/// no `reverse_proxy.stream` block is configured. +fn reentry_ttl_for_stream(stream: Option<&tuliprox_core::model::StreamConfig>) -> std::time::Duration { + stream.map_or(std::time::Duration::from_millis(DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS), |stream| { + stream.recent_eviction_reentry_ttl + }) +} + +pub(crate) fn get_reentry_ttl(adm: &AdmissionCtx) -> std::time::Duration { + let config = adm.app_config.config.load(); + reentry_ttl_for_stream(config.reverse_proxy.as_ref().and_then(|rp| rp.stream.as_ref())) +} #[derive(Clone, Copy)] pub enum EvictionReentryGuard<'a> { @@ -120,10 +132,10 @@ pub async fn resolve_playback_request_admission( adm.active_users.terminate_session(&user.username, session.token.as_str()).await; } return ( - crate::ConnectionAdmission { - permission: UserConnectionPermission::Exhausted, - kind: user_session.and_then(|session| session.connection_kind).or(Some(crate::ConnectionKind::Normal)), - }, + crate::ConnectionAdmission::exhausted( + crate::AdmissionRejectionReason::UserConnectionsExhausted, + user_session.and_then(|session| session.connection_kind).or(Some(crate::ConnectionKind::Normal)), + ), None, request_class, ); @@ -133,10 +145,9 @@ pub async fn resolve_playback_request_admission( // running strategies or modifying counted state. Caller handles the actual activation. if request_class == PlaybackRequestClass::Prepare { return ( - crate::ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: user_session.and_then(|session| session.connection_kind).or(Some(crate::ConnectionKind::Normal)), - }, + crate::ConnectionAdmission::allowed( + user_session.and_then(|session| session.connection_kind).or(Some(crate::ConnectionKind::Normal)), + ), None, request_class, ); @@ -144,10 +155,10 @@ pub async fn resolve_playback_request_admission( if request_class == PlaybackRequestClass::FollowUp || !limits_enabled { return ( - crate::ConnectionAdmission { - permission: user_session.map_or(UserConnectionPermission::Allowed, |session| session.permission), - kind: user_session.and_then(|session| session.connection_kind).or(Some(crate::ConnectionKind::Normal)), - }, + crate::ConnectionAdmission::from_permission( + user_session.map_or(UserConnectionPermission::Allowed, |session| session.permission), + user_session.and_then(|session| session.connection_kind).or(Some(crate::ConnectionKind::Normal)), + ), None, request_class, ); @@ -177,7 +188,12 @@ pub async fn resolve_playback_request_admission( // // Only the strategy path emits. An explicit `Terminate` also resolves to // `Exhausted`, but that is a requested teardown, not a denial. - if result.admission.permission == UserConnectionPermission::Exhausted { + // + // Background retry of a recently evicted stream is quietly suppressed to avoid + // playback ping-pong. Count it as a diagnostic, never as a connection denial. + if result.admission.is_reentry_suppressed() { + adm.active_users.record_reentry_suppressed(); + } else if result.admission.permission() == UserConnectionPermission::Exhausted { adm.active_users.events().send_event(EventMessage::ConnectionDenied(ConnectionDenied::new( Arc::from(user.username.as_str()), Arc::from(fingerprint.client_ip.as_str()), @@ -192,19 +208,22 @@ pub async fn resolve_playback_request_admission( async fn should_suppress_eviction_for_recent_request( adm: &AdmissionCtx, request: &AdmissionRequest<'_>, - target_addr: &std::net::SocketAddr, + target: &crate::EvictionTarget, ) -> bool { match request.eviction_reentry_guard { EvictionReentryGuard::Session(session_token) => adm .active_users .recently_evicted_session_protected_addr(session_token) .await - .is_some_and(|protected_addr| protected_addr == *target_addr), - EvictionReentryGuard::SocketPlayback { virtual_id } => adm - .active_users - .recent_socket_reentry_protected_addr(request.username, request.client_ip, virtual_id) - .await - .is_some_and(|protected_addr| protected_addr == *target_addr), + .is_some_and(|protected_addr| protected_addr == target.addr), + EvictionReentryGuard::SocketPlayback { virtual_id } => { + target.virtual_id != virtual_id + && adm + .active_users + .recent_socket_reentry_protected_addr(request.username, request.client_ip, virtual_id) + .await + .is_some_and(|protected_addr| protected_addr == target.addr) + } } } @@ -263,14 +282,14 @@ pub fn get_effective_admission_strategies(adm: &AdmissionCtx) -> Arc<[AdmissionS } } +enum StrategyLoopResult { + Admitted(AdmissionStrategyResolution), + Rejected(crate::AdmissionRejectionReason), +} + /// Shared strategy-evaluation loop used by both the initial admission path /// (`resolve_admission_with_strategies`) and the remaining-strategies path /// (`evaluate_remaining_strategies_after_grace`). -/// -/// Returns `Some(resolution)` when a Grace or a successful Eviction+Retry is found. -/// Returns `None` when every strategy in `strategies` returns `NoMatch` β€” the caller -/// is then responsible for constructing the final exhausted result with the correct -/// `kind` (preserved from the original admission). async fn evaluate_admission_strategy_loop( adm: &AdmissionCtx, request: &AdmissionRequest<'_>, @@ -278,7 +297,7 @@ async fn evaluate_admission_strategy_loop( base_idx: usize, admission: crate::ConnectionAdmission, build_grace_ctx: F, -) -> Option +) -> StrategyLoopResult where F: Fn(usize) -> GraceResolutionContext, { @@ -292,6 +311,12 @@ where // kick that frees nothing is taken as evidence that the next one would not // help either, and later eviction strategies are skipped. let mut evictions_ineffective = false; + // Set when a candidate was skipped because the reentry guard protects it. + let mut suppressed_candidate = false; + // Set only after a *non-protected* candidate was actually evicted. A genuine + // eviction means the final rejection is real resource exhaustion, not a + // reentry suppression, even if an earlier candidate was protected. + let mut performed_legitimate_eviction = false; // `enumerate` rather than a manual counter: the suppressed-eviction arm below // uses `continue`, which used to skip a trailing `idx += 1` and hand every @@ -307,11 +332,8 @@ where // Return a FRESH admission with GracePeriod permission (not the admission // parameter, which may have Exhausted permission). The kind is preserved from // the original admission. - return Some(AdmissionStrategyResolution { - admission: crate::ConnectionAdmission { - permission: UserConnectionPermission::GracePeriod, - kind: admission.kind, - }, + return StrategyLoopResult::Admitted(AdmissionStrategyResolution { + admission: crate::ConnectionAdmission::grace_period(admission.kind()), grace_mode: Some(mode), grace_context: Some(build_grace_ctx(base_idx + idx)), }); @@ -325,22 +347,25 @@ where ); continue; } - if should_suppress_eviction_for_recent_request(adm, request, &target.addr).await { + if should_suppress_eviction_for_recent_request(adm, request, &target).await { debug!( "Skipping eviction strategy {strategy:?} for recently evicted request of user {username} targeting {}", target.addr ); + suppressed_candidate = true; + // Exclude this specific stream candidate so subsequent strategies can evaluate other candidates + candidates.retain(|c| c.uid != target.uid); continue; } debug!("Evicting connection {} for user {username}", target.addr); let connections_before = adm.active_users.user_connections(username).await; - adm.active_users - .mark_recent_eviction_guard_for_addr(&target.addr, *request_addr, RECENT_EVICTION_REENTRY_TTL_SECS) - .await; + let ttl = get_reentry_ttl(adm); + adm.active_users.mark_recent_eviction_guard_for_addr(&target.addr, *request_addr, ttl).await; adm.connection_manager.release_connection_as_kicked(&target.addr).await; + performed_legitimate_eviction = true; let retry_admission = get_admission_for_request(adm, request).await; - if retry_admission.permission == UserConnectionPermission::Allowed { - return Some(AdmissionStrategyResolution { + if retry_admission.permission() == UserConnectionPermission::Allowed { + return StrategyLoopResult::Admitted(AdmissionStrategyResolution { admission: retry_admission, grace_mode: None, grace_context: None, @@ -362,8 +387,24 @@ where } } - // All strategies returned NoMatch β€” caller constructs the final exhausted result. - None + StrategyLoopResult::Rejected(rejection_after_strategy_loop(suppressed_candidate, performed_legitimate_eviction)) +} + +/// Classifies the rejection after the strategy loop. +/// +/// `RecentEvictionReentry` is only correct when the request failed *solely* because +/// every candidate was reentry-protected and no legitimate eviction was performed. +/// Once a non-protected candidate was evicted and admission still fails, the request +/// hit genuine resource exhaustion and must be reported as such. +fn rejection_after_strategy_loop( + suppressed_candidate: bool, + performed_legitimate_eviction: bool, +) -> crate::AdmissionRejectionReason { + if suppressed_candidate && !performed_legitimate_eviction { + crate::AdmissionRejectionReason::RecentEvictionReentry + } else { + crate::AdmissionRejectionReason::UserConnectionsExhausted + } } pub async fn resolve_admission_with_strategies( @@ -375,14 +416,21 @@ pub async fn resolve_admission_with_strategies( let username = request.username; let admission = get_admission_for_request(adm, &request).await; - if admission.permission != UserConnectionPermission::Exhausted { + if admission.permission() != UserConnectionPermission::Exhausted { return AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None }; } let strategies = get_effective_admission_strategies(adm); if strategies.is_empty() { debug!("No admission strategies configured, denying request for user {username}"); - return AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None }; + return AdmissionStrategyResolution { + admission: crate::ConnectionAdmission::exhausted( + crate::AdmissionRejectionReason::UserConnectionsExhausted, + admission.kind(), + ), + grace_mode: None, + grace_context: None, + }; } let _admission_guard = adm.active_users.acquire_user_admission(username).await; @@ -394,7 +442,7 @@ pub async fn resolve_admission_with_strategies( // free. let admission = get_admission_for_request(adm, &request).await; - if admission.permission != UserConnectionPermission::Exhausted { + if admission.permission() != UserConnectionPermission::Exhausted { debug!("Admission became available while waiting on the admission gate for user {username}"); return AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None }; } @@ -402,17 +450,20 @@ pub async fn resolve_admission_with_strategies( let build_grace_ctx = |global_idx: usize| GraceResolutionContext { strategy_index: global_idx, strategies: strategies.clone(), - kind: admission.kind, + kind: admission.kind(), }; - if let Some(resolution) = - evaluate_admission_strategy_loop(adm, &request, &strategies, 0, admission, build_grace_ctx).await - { - return resolution; + match evaluate_admission_strategy_loop(adm, &request, &strategies, 0, admission, build_grace_ctx).await { + StrategyLoopResult::Admitted(resolution) => resolution, + StrategyLoopResult::Rejected(reason) => { + debug!("No admission strategy could admit user {username}: {reason:?}"); + AdmissionStrategyResolution { + admission: crate::ConnectionAdmission::exhausted(reason, admission.kind()), + grace_mode: None, + grace_context: None, + } + } } - - debug!("No admission strategy could admit user {username}"); - AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None } } /// Evaluates only the strategies that come AFTER the already-used grace strategy. @@ -433,18 +484,16 @@ pub async fn evaluate_remaining_strategies_after_grace( grace_context: &GraceResolutionContext, original_kind: Option, ) -> AdmissionStrategyResolution { - use shared::model::UserConnectionPermission; - let username = request.username; let remaining = grace_context.strategy_index + 1; let strategies = &grace_context.strategies; if remaining >= strategies.len() { debug!("No remaining strategies after grace for user {username}"); return AdmissionStrategyResolution { - admission: crate::ConnectionAdmission { - permission: UserConnectionPermission::Exhausted, - kind: original_kind, - }, + admission: crate::ConnectionAdmission::exhausted( + crate::AdmissionRejectionReason::UserConnectionsExhausted, + original_kind, + ), grace_mode: None, grace_context: None, }; @@ -454,7 +503,8 @@ pub async fn evaluate_remaining_strategies_after_grace( // returned `ConnectionAdmission`, and `build_grace_ctx` copies it onto the // `GraceResolutionContext`. Seeding it with `original_kind` keeps every exit from this // function reporting the kind the original admission decided. - let admission = crate::ConnectionAdmission { permission: UserConnectionPermission::Exhausted, kind: original_kind }; + let admission = + crate::ConnectionAdmission::exhausted(crate::AdmissionRejectionReason::UserConnectionsExhausted, original_kind); let build_grace_ctx = |global_idx: usize| GraceResolutionContext { strategy_index: global_idx, strategies: strategies.clone(), @@ -463,17 +513,73 @@ pub async fn evaluate_remaining_strategies_after_grace( let _admission_guard = adm.active_users.acquire_user_admission(username).await; - if let Some(resolution) = - evaluate_admission_strategy_loop(adm, &request, &strategies[remaining..], remaining, admission, build_grace_ctx) - .await + match evaluate_admission_strategy_loop( + adm, + &request, + &strategies[remaining..], + remaining, + admission, + build_grace_ctx, + ) + .await { - return resolution; - } - - debug!("No remaining strategy could admit user {username}"); - AdmissionStrategyResolution { - admission: crate::ConnectionAdmission { permission: UserConnectionPermission::Exhausted, kind: original_kind }, - grace_mode: None, - grace_context: None, + StrategyLoopResult::Admitted(resolution) => resolution, + StrategyLoopResult::Rejected(reason) => { + debug!("No remaining strategy could admit user {username}: {reason:?}"); + AdmissionStrategyResolution { + admission: crate::ConnectionAdmission::exhausted(reason, original_kind), + grace_mode: None, + grace_context: None, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{AdmissionRejectionReason, ConnectionAdmission}; + use shared::model::UserConnectionPermission; + + #[test] + fn reentry_ttl_uses_configured_value_and_falls_back_to_default() { + assert_eq!( + reentry_ttl_for_stream(None), + std::time::Duration::from_millis(DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS) + ); + let stream = tuliprox_core::model::StreamConfig { + recent_eviction_reentry_ttl: std::time::Duration::from_millis(2_500), + ..tuliprox_core::model::StreamConfig::default() + }; + assert_eq!(reentry_ttl_for_stream(Some(&stream)), std::time::Duration::from_millis(2_500)); + } + + #[test] + fn rejection_classification_distinguishes_reentry_from_exhaustion() { + // Pure reentry: a protected candidate was skipped and nothing was evicted. + assert_eq!(rejection_after_strategy_loop(true, false), AdmissionRejectionReason::RecentEvictionReentry); + // Mixed case: a protected candidate was skipped, but a legitimate eviction was + // performed and admission still failed. That residual failure is genuine + // exhaustion, not a quiet suppression. + assert_eq!(rejection_after_strategy_loop(true, true), AdmissionRejectionReason::UserConnectionsExhausted); + assert_eq!(rejection_after_strategy_loop(false, true), AdmissionRejectionReason::UserConnectionsExhausted); + assert_eq!(rejection_after_strategy_loop(false, false), AdmissionRejectionReason::UserConnectionsExhausted); + } + + #[test] + fn exhausted_admission_always_carries_a_reason() { + let exhausted = ConnectionAdmission::from_permission(UserConnectionPermission::Exhausted, None); + assert_eq!(exhausted.permission(), UserConnectionPermission::Exhausted); + assert_eq!(exhausted.rejection_reason(), Some(AdmissionRejectionReason::UserConnectionsExhausted)); + assert!(!exhausted.is_reentry_suppressed()); + + assert_eq!( + ConnectionAdmission::from_permission(UserConnectionPermission::Allowed, None).rejection_reason(), + None + ); + assert_eq!( + ConnectionAdmission::from_permission(UserConnectionPermission::GracePeriod, None).permission(), + UserConnectionPermission::GracePeriod + ); } } diff --git a/backend/session/src/admission_strategy.rs b/backend/session/src/admission_strategy.rs index 14dc96dd4..23f2f3da5 100644 --- a/backend/session/src/admission_strategy.rs +++ b/backend/session/src/admission_strategy.rs @@ -1,5 +1,5 @@ use log::debug; -use shared::model::AdmissionStrategy; +use shared::model::{AdmissionStrategy, VirtualId}; use std::{net::SocketAddr, sync::Arc}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -18,6 +18,8 @@ pub enum GraceMode { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct EvictionTarget { pub addr: SocketAddr, + pub virtual_id: VirtualId, + pub uid: u32, } #[derive(Debug, Clone)] @@ -88,14 +90,22 @@ fn evaluate_evict_same_ip( } match selected { - Some(candidate) => AdmissionDecision::Evict(EvictionTarget { addr: candidate.addr }), + Some(candidate) => AdmissionDecision::Evict(EvictionTarget { + addr: candidate.addr, + virtual_id: candidate.virtual_id, + uid: candidate.uid, + }), None => AdmissionDecision::NoMatch, } } fn evaluate_evict_user(candidates: &[EvictionCandidate], order: EvictionOrder) -> AdmissionDecision { match select_candidate(candidates.iter(), order) { - Some(candidate) => AdmissionDecision::Evict(EvictionTarget { addr: candidate.addr }), + Some(candidate) => AdmissionDecision::Evict(EvictionTarget { + addr: candidate.addr, + virtual_id: candidate.virtual_id, + uid: candidate.uid, + }), None => AdmissionDecision::NoMatch, } } @@ -109,8 +119,8 @@ fn select_candidate<'a>( let should_replace = match selected { None => true, Some(current) => match order { - EvictionOrder::Oldest => candidate.ts < current.ts, - EvictionOrder::Latest => candidate.ts > current.ts, + EvictionOrder::Oldest => (candidate.ts, candidate.uid) < (current.ts, current.uid), + EvictionOrder::Latest => (candidate.ts, candidate.uid) > (current.ts, current.uid), }, }; if should_replace { @@ -124,7 +134,10 @@ fn select_candidate<'a>( pub struct EvictionCandidate { pub addr: SocketAddr, pub client_ip: String, + pub virtual_id: VirtualId, pub ts: u64, + /// Monotonic stream identity used to order streams created in the same second. + pub uid: u32, } #[cfg(test)] @@ -135,7 +148,13 @@ mod tests { fn addr(port: u16) -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port) } fn candidate(port: u16, ip: &str, ts: u64) -> EvictionCandidate { - EvictionCandidate { addr: addr(port), client_ip: ip.to_string(), ts } + EvictionCandidate { + addr: addr(port), + client_ip: ip.to_string(), + virtual_id: VirtualId::new(u32::from(port)), + ts, + uid: u32::from(port), + } } #[test] @@ -183,6 +202,13 @@ mod tests { assert_evict(AdmissionStrategy::EvictUserLatest, &candidates, 1000); } + #[test] + fn equal_timestamp_uses_monotonic_stream_identity() { + let candidates = vec![candidate(1000, "1.1.1.1", 100), candidate(1001, "1.1.1.1", 100)]; + assert_evict(AdmissionStrategy::EvictUserSameIpOldest, &candidates, 1000); + assert_evict(AdmissionStrategy::EvictUserSameIpLatest, &candidates, 1001); + } + #[test] fn no_same_ip_candidate_returns_no_match() { let ctx = StrategyContext { username: "user1", client_ip: "1.1.1.1" }; diff --git a/backend/session/src/connection_manager.rs b/backend/session/src/connection_manager.rs index 21168ac3f..68c9379a3 100644 --- a/backend/session/src/connection_manager.rs +++ b/backend/session/src/connection_manager.rs @@ -1,8 +1,9 @@ use crate::{ uses_direct_body_idle_timeout, ActiveProviderManager, ActiveUserConnectionParams, ActiveUserManager, EventManager, - SharedStreamManager, + ManagedProviderHandle, SharedStreamManager, }; use arc_swap::ArcSwapOption; +use futures::future::BoxFuture; use log::{debug, warn}; use shared::{ model::{ @@ -14,35 +15,63 @@ use shared::{ use std::{ borrow::Cow, cmp::Reverse, - collections::{BinaryHeap, HashMap, VecDeque}, + collections::{BinaryHeap, HashMap, HashSet, VecDeque}, net::SocketAddr, str::FromStr, sync::{ - atomic::{AtomicU32, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, Arc, Mutex, MutexGuard, }, thread, time::Duration, }; -use tokio::sync::{mpsc, Notify}; +use tokio::sync::{mpsc, Notify, RwLock, RwLockReadGuard}; +use tokio_util::sync::CancellationToken; use tuliprox_core::{ - model::{DisconnectQos, Fingerprint, ProviderHandle, StreamHistoryConfig, StreamHistoryRecord}, + model::{ + DisconnectQos, Fingerprint, PlaybackRequestOutcome, ProviderHandle, SharedSubscriberId, StreamHistoryConfig, + StreamHistoryRecord, + }, utils::debug_if_enabled, }; use tuliprox_repository::{recover_pending_files, StreamHistoryWriter}; // Maximum number of deferred cleanup actions buffered before producers must wait/drop. const CLEANUP_QUEUE_CAPACITY: usize = 4096; +// Reserved lane for control work (HLS origin I/O, shutdown) so a saturated body-cleanup +// queue cannot starve the control path that releases provider handles. +const CONTROL_CLEANUP_CAPACITY: usize = 64; pub const PROVIDER_END_NOT_SET: u8 = 0; pub const PROVIDER_END_CLOSED: u8 = 1; // Provider EOF pub const PROVIDER_END_ERROR: u8 = 2; // Provider Err +pub const PROVIDER_END_PREEMPTED: u8 = 3; // Preempted by higher priority const PREEMPT_REENTRY_BLOCK_SECS: u64 = 3; +// Bounded wait for a mandatory cleanup admission right before a request registration. +const CLEANUP_ADMISSION_TIMEOUT: Duration = Duration::from_secs(5); // Rebuild the expiry heap when it grows beyond this multiple of the live index size. const SOCKET_EXPIRY_QUEUE_REBUILD_FACTOR: usize = 2; // Avoid rebuilding the expiry heap unless it contains at least this many stale entries. const SOCKET_EXPIRY_QUEUE_REBUILD_MIN_STALE: usize = 256; fn notify_capacity(capacity_notify: &Notify) { capacity_notify.notify_waiters(); } +/// Proof that a shared-stream response owns the guaranteed terminal cleanup permit for +/// its subscriber. The constructor is crate-private, so callers outside this crate cannot +/// fabricate the claim; it is only produced when a cleanup permit is actually reserved. +/// Each registration consumes one capability, so a stale capability cannot be duplicated +/// and reused after its owner has released. +#[derive(Debug)] +pub struct SharedCleanupCapability { + subscriber_id: SharedSubscriberId, +} + +impl SharedCleanupCapability { + pub(crate) fn new(subscriber_id: SharedSubscriberId) -> Self { Self { subscriber_id } } + + pub fn stream_uid(&self) -> u32 { self.subscriber_id.stream_uid() } + + pub fn subscriber_id(&self) -> SharedSubscriberId { self.subscriber_id } +} + struct BackpressureState { overflow: VecDeque, draining: bool, @@ -181,32 +210,31 @@ fn lock_backpressure_state(state: &Mutex>) -> MutexGuard #[derive(Clone)] struct SocketActivityTracker { - pending: Arc>>, + pending: Arc>>, notify: Arc, } impl SocketActivityTracker { - fn new() -> Self { Self { pending: Arc::new(Mutex::new(HashMap::new())), notify: Arc::new(Notify::new()) } } + fn new() -> Self { Self { pending: Arc::new(Mutex::new(HashSet::new())), notify: Arc::new(Notify::new()) } } fn track(&self, event: SocketActivityEvent) { - let key = event.addr(); let mut pending = lock_socket_activity_pending(self.pending.as_ref()); - pending.insert(key, event); + pending.insert(event); drop(pending); self.notify.notify_one(); } fn drain(&self) -> Vec { let mut pending = lock_socket_activity_pending(self.pending.as_ref()); - pending.drain().map(|(_, event)| event).collect() + pending.drain().collect() } async fn notified(&self) { self.notify.notified().await; } } fn lock_socket_activity_pending( - pending: &Mutex>, -) -> MutexGuard<'_, HashMap> { + pending: &Mutex>, +) -> MutexGuard<'_, HashSet> { pending.lock().unwrap_or_else(|poisoned| { warn!("Socket activity state was poisoned, continuing with recovered state"); poisoned.into_inner() @@ -223,7 +251,17 @@ struct CleanupWorkerDeps { } pub enum CleanupEvent { + ReleaseSharedSubscriber { + addr: SocketAddr, + subscriber_id: SharedSubscriberId, + request_id: Option, + owner: Option>, + }, ReleaseStream { + request_id: Option, + /// Playback owner (session token) captured at acquire, so the provider request + /// can be finished independently of whether the user claim still exists. + owner: Option>, addr: SocketAddr, stream_uid: Option, provider_end_reason: u8, @@ -238,6 +276,7 @@ pub enum CleanupEvent { handle: Option, }, ReleaseStreamAndProviderHandle { + request_id: Option, addr: SocketAddr, stream_uid: Option, handle: Option, @@ -248,16 +287,21 @@ pub enum CleanupEvent { }, UpdateDetailAndReleaseProvider { addr: SocketAddr, + stream_uid: Option, video_type: CustomVideoStreamType, handle: Option, }, - UpdateDetailAndReleaseProviderConnection { - addr: SocketAddr, - video_type: CustomVideoStreamType, - }, AdaptiveSessionExpired { stream_info: Box, }, + /// Confirms that real provider media reached the client for a playback lease. + /// Only a confirmed lease may reserve provider capacity against other playbacks. + ConfirmPlaybackLease { + owner: Arc, + request_id: Option, + }, + /// Runs a deferred asynchronous cleanup task in the managed cleanup worker. + Defer(BoxFuture<'static, ()>), } async fn handle_release_connection(deps: &CleanupWorkerDeps, addr: SocketAddr) { @@ -295,7 +339,7 @@ async fn release_connection_parts( if matches!(reason, DisconnectReason::ClientKicked) { for stream_info in &removed.removed_streams { if let Some(session_token) = stream_info.session_token.as_deref() { - deps.provider_manager.clear_provider_reservation(session_token).await; + deps.provider_manager.clear_provider_reservation(session_token); } } // Explicitly terminate all sessions for the kicked addr. This expires them @@ -318,7 +362,7 @@ async fn release_connection_parts( None, ); } - deps.provider_manager.release_connection(addr).await; + deps.provider_manager.release_connection(addr); deps.shared_stream_manager.release_connection(addr, send_shared_stop_signal).await; if removed.addr_removed && !removed.removed_streams.is_empty() { deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Disconnected(*addr))); @@ -326,6 +370,7 @@ async fn release_connection_parts( notify_capacity(deps.capacity_notify.as_ref()); } +#[allow(clippy::too_many_arguments)] async fn handle_release_stream( deps: &CleanupWorkerDeps, addr: SocketAddr, @@ -334,6 +379,8 @@ async fn handle_release_stream( reconnect_count: u8, provider_error_class: Option<&'static str>, provider_http_status: Option, + request_id: Option, + owner: Option>, ) { if let Some(stream_info) = release_stream_with_disconnect( deps, @@ -343,6 +390,8 @@ async fn handle_release_stream( reconnect_count, provider_error_class, provider_http_status, + request_id, + owner, ) .await { @@ -354,9 +403,9 @@ async fn handle_release_stream( } } -async fn handle_release_provider_handle(deps: &CleanupWorkerDeps, handle: Option) { +fn handle_release_provider_handle(deps: &CleanupWorkerDeps, handle: Option) { if let Some(handle) = handle { - deps.provider_manager.release_handle(&handle).await; + deps.provider_manager.release_handle(&handle); notify_capacity(deps.capacity_notify.as_ref()); } } @@ -371,9 +420,10 @@ async fn handle_release_stream_and_provider_handle( reconnect_count: u8, provider_error_class: Option<&'static str>, provider_http_status: Option, + request_id: Option, ) { let provider_released = if let Some(handle) = handle { - deps.provider_manager.release_handle(&handle).await; + deps.provider_manager.release_handle(&handle); true } else { false @@ -386,6 +436,8 @@ async fn handle_release_stream_and_provider_handle( reconnect_count, provider_error_class, provider_http_status, + request_id, + None, ) .await; if let Some(stream_info) = stream_released.as_ref() { @@ -404,12 +456,18 @@ async fn handle_update_detail_and_release_provider( addr: SocketAddr, video_type: CustomVideoStreamType, handle: Option, + stream_uid: Option, ) { - if let Some(stream_info) = deps.user_manager.update_stream_detail(&addr, video_type).await { + let stream_info = if let Some(uid) = stream_uid { + deps.user_manager.update_stream_detail_by_uid(uid, video_type).await + } else { + deps.user_manager.update_stream_detail(&addr, video_type).await + }; + if let Some(stream_info) = stream_info { if matches!(video_type, CustomVideoStreamType::LowPriorityPreempted) { deps.user_manager - .block_user_for_stream( - &addr, + .block_user_for_stream_uid( + stream_info.uid, shared::model::VirtualId::new(stream_info.channel.virtual_id), PREEMPT_REENTRY_BLOCK_SECS, ) @@ -418,33 +476,11 @@ async fn handle_update_detail_and_release_provider( deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); } if let Some(handle) = handle { - deps.provider_manager.release_handle(&handle).await; + deps.provider_manager.release_handle(&handle); notify_capacity(deps.capacity_notify.as_ref()); } } -async fn handle_update_detail_and_release_provider_connection( - deps: &CleanupWorkerDeps, - addr: SocketAddr, - video_type: CustomVideoStreamType, -) { - if let Some(stream_info) = deps.user_manager.update_stream_detail(&addr, video_type).await { - if matches!(video_type, CustomVideoStreamType::LowPriorityPreempted) { - deps.user_manager - .block_user_for_stream( - &addr, - shared::model::VirtualId::new(stream_info.channel.virtual_id), - PREEMPT_REENTRY_BLOCK_SECS, - ) - .await; - } - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); - } - deps.provider_manager.release_connection(&addr).await; - deps.shared_stream_manager.release_connection(&addr, false).await; - notify_capacity(deps.capacity_notify.as_ref()); -} - async fn handle_adaptive_session_expired(deps: &CleanupWorkerDeps, stream_info: Box) { let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; let bytes_sent = qos.map(|qos| qos.bytes_total); @@ -465,6 +501,7 @@ async fn handle_adaptive_session_expired(deps: &CleanupWorkerDeps, stream_info: notify_capacity(deps.capacity_notify.as_ref()); } +#[allow(clippy::too_many_arguments)] async fn release_stream_with_disconnect( deps: &CleanupWorkerDeps, addr: SocketAddr, @@ -473,35 +510,75 @@ async fn release_stream_with_disconnect( reconnect_count: u8, provider_error_class: Option<&'static str>, provider_http_status: Option, + request_id: Option, + owner: Option>, ) -> Option { - let stream_info = if let Some(stream_uid) = stream_uid { - deps.user_manager.release_stream_by_uid(&addr, stream_uid).await + let detach = if let Some(stream_uid) = stream_uid { + deps.user_manager.release_stream_request_by_uid(&addr, stream_uid).await } else { - deps.user_manager.release_stream(&addr).await + crate::StreamRequestDetach::NotFound }; - let Some(stream_info) = stream_info else { - debug_if_enabled!( - "Stream release skipped: no active stream for {} uid={:?}", - sanitize_sensitive_info(&addr.to_string()), - stream_uid - ); - return None; - }; - let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; - let bytes_sent = qos.map(|qos| qos.bytes_total); - let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); - deps.event_manager.unregister_meter_client(stream_info.uid).await; - let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); - let provider_reconnect_count = (reconnect_count > 0).then_some(reconnect_count); - emit_disconnect_record( - &deps.history_writer, - &stream_info, - reason, - &DisconnectQos { bytes_sent, first_byte_latency_ms, provider_reconnect_count }, - provider_error_class, - provider_http_status, - ); - Some(stream_info) + match detach { + crate::StreamRequestDetach::NotFound => { + debug_if_enabled!( + "Stream release skipped: no active stream for {} uid={:?}", + sanitize_sensitive_info(&addr.to_string()), + stream_uid + ); + // The provider request must still be finished even when the user claim is + // already gone: the identity travels with the cleanup event. + if let (Some(owner), Some(request_id)) = (owner.as_deref(), request_id) { + let reason = resolve_disconnect_reason_from_provider_end(provider_end_reason); + deps.provider_manager.finish_identified_playback_request( + owner, + request_id, + playback_outcome_for_reason(reason), + ); + notify_capacity(deps.capacity_notify.as_ref()); + } + None + } + crate::StreamRequestDetach::Retained(stream_info) | crate::StreamRequestDetach::Preserved(stream_info) => { + let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); + if let (Some(session_token), Some(request_id)) = + (owner.as_deref().or(stream_info.session_token.as_deref()), request_id) + { + deps.provider_manager.finish_identified_playback_request( + session_token, + request_id, + playback_outcome_for_reason(reason), + ); + } + notify_capacity(deps.capacity_notify.as_ref()); + None + } + crate::StreamRequestDetach::Removed(stream_info) => { + let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; + let bytes_sent = qos.map(|qos| qos.bytes_total); + let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); + deps.event_manager.unregister_meter_client(stream_info.uid).await; + let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); + if let (Some(session_token), Some(request_id)) = + (owner.as_deref().or(stream_info.session_token.as_deref()), request_id) + { + deps.provider_manager.finish_identified_playback_request( + session_token, + request_id, + playback_outcome_for_reason(reason), + ); + } + let provider_reconnect_count = (reconnect_count > 0).then_some(reconnect_count); + emit_disconnect_record( + &deps.history_writer, + &stream_info, + reason, + &DisconnectQos { bytes_sent, first_byte_latency_ms, provider_reconnect_count }, + provider_error_class, + provider_http_status, + ); + Some(stream_info) + } + } } #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] @@ -510,7 +587,7 @@ struct SocketExpiryEntry { addr: SocketAddr, } -#[derive(Clone, Debug)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] enum SocketActivityEvent { HttpActivity { addr: SocketAddr }, DirectBodyActivity { addr: SocketAddr }, @@ -536,10 +613,15 @@ pub struct ConnectionManager { event_manager: Arc, close_socket_signal_tx: tokio::sync::broadcast::Sender, cleanup_sender: BackpressureSender, + control_cleanup_tx: mpsc::Sender, socket_activity_tracker: SocketActivityTracker, capacity_notify: Arc, stream_uid_counter: AtomicU32, history_writer: Arc>, + is_shutting_down: AtomicBool, + admission_gate: RwLock<()>, + shutdown_token: CancellationToken, + worker_handles: std::sync::Mutex>>, } pub struct ConnectionParams<'a> { @@ -563,6 +645,154 @@ pub enum ConnectionHistoryMode { RefreshOnly, } +/// Guaranteed final cleanup for a registered request, transferred into the body. +/// +/// The permit was reserved before the registration mutation, so the release is +/// delivered even under cleanup-queue pressure. The body owns this and calls +/// [`OwnedRequestCleanup::finish`] exactly once with the real provider outcome; an +/// un-polled body drops it and gets a conservative release instead of a successful EOF. +pub struct OwnedRequestCleanup { + addr: SocketAddr, + request_uid: u32, + /// Provider request identity captured at acquire, so an automatic rollback can + /// finish the provider lease even when the user claim was never fully registered. + provider_request_id: Option, + /// Playback owner (session token) captured at acquire. The provider finish must not + /// depend on the user claim still existing, so the owner travels with the cleanup. + owner: Option>, + permit: Option>, + finished: bool, +} + +impl OwnedRequestCleanup { + /// Releases the reserved cleanup right without emitting any release. + fn disarm(&mut self) { + self.finished = true; + self.permit = None; + } + + /// Releases the request claim exactly once with the actual provider outcome. + #[allow(clippy::too_many_arguments)] + pub fn finish( + &mut self, + request_id: Option, + provider_end_reason: u8, + reconnect_count: u8, + provider_error_class: Option<&'static str>, + provider_http_status: Option, + ) { + if self.finished { + return; + } + self.finished = true; + let Some(permit) = self.permit.take() else { + return; + }; + permit.send(CleanupEvent::ReleaseStream { + request_id, + owner: self.owner.clone(), + addr: self.addr, + stream_uid: Some(self.request_uid), + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + }); + } +} + +impl Drop for OwnedRequestCleanup { + fn drop(&mut self) { + // Conservative fallback for a body that is dropped without an explicit finish: + // never invent a successful EOF, but still finish the provider request with its + // captured identity so an abandoned start does not linger until the startup TTL. + self.finish(self.provider_request_id, PROVIDER_END_NOT_SET, 0, None, None); + } +} + +/// Why a playback request was rejected before a body could be created. Carried on +/// `RegisteredPlaybackRequest` so the caller can surface a non-success HTTP response +/// instead of silently dropping the request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectionRejectionReason { + CleanupReceiverClosed, + CleanupAdmissionTimeout, + RegistrationFailed, +} + +impl std::fmt::Display for ConnectionRejectionReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::CleanupReceiverClosed => write!(f, "cleanup receiver closed"), + Self::CleanupAdmissionTimeout => write!(f, "cleanup admission timed out"), + Self::RegistrationFailed => write!(f, "connection registration failed"), + } + } +} + +impl std::error::Error for ConnectionRejectionReason {} + +pub struct RegisteredPlaybackRequest { + pub request_uid: u32, + pub display_stream: Option, + cleanup: Option, + rejection: Option, +} + +impl std::fmt::Debug for RegisteredPlaybackRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RegisteredPlaybackRequest") + .field("request_uid", &self.request_uid) + .field("display_stream", &self.display_stream) + .field("rejection", &self.rejection) + .finish_non_exhaustive() + } +} + +impl RegisteredPlaybackRequest { + #[inline] + pub fn new(request_uid: u32, display_stream: Option) -> Self { + Self { request_uid, display_stream, cleanup: None, rejection: None } + } + + #[inline] + fn rejected(request_uid: u32, reason: ConnectionRejectionReason) -> Self { + Self { request_uid, display_stream: None, cleanup: None, rejection: Some(reason) } + } + + #[inline] + fn with_cleanup( + request_uid: u32, + display_stream: Option, + cleanup: Option, + ) -> Self { + Self { request_uid, display_stream, cleanup, rejection: None } + } + + /// Marks the request as taken over by a path without a body (compatibility API or + /// shared source), disabling rollback on drop. + #[inline] + pub fn commit(&mut self) { + if let Some(mut cleanup) = self.cleanup.take() { + cleanup.disarm(); + } + } + + /// Transfers the guaranteed cleanup right into the body. The body owns the returned + /// value and must finish it exactly once with the real provider outcome. + #[inline] + pub fn into_body_cleanup(&mut self) -> Option { self.cleanup.take() } + + #[inline] + pub fn display_uid(&self) -> Option { self.display_stream.as_ref().map(|s| s.uid) } + + #[inline] + pub fn rejection_reason(&self) -> Option { self.rejection } + + #[inline] + pub fn into_display_stream(self) -> Option { self.display_stream } +} + impl ConnectionManager { pub fn new( user_manager: &Arc, @@ -571,49 +801,90 @@ impl ConnectionManager { event_manager: &Arc, history_config: Option<&StreamHistoryConfig>, ) -> Self { + Self::new_with_capacity( + user_manager, + provider_manager, + shared_stream_manager, + event_manager, + history_config, + CLEANUP_QUEUE_CAPACITY, + ) + } + + /// Same as [`ConnectionManager::new`], but with an explicit cleanup-queue capacity + /// derived from configuration instead of the hard-coded default. + pub fn new_with_capacity( + user_manager: &Arc, + provider_manager: &Arc, + shared_stream_manager: &Arc, + event_manager: &Arc, + history_config: Option<&StreamHistoryConfig>, + cleanup_capacity: usize, + ) -> Self { + let cleanup_capacity = cleanup_capacity.max(1); let history_writer = Arc::new(ArcSwapOption::new(build_history_writer(history_config))); let (close_socket_signal_tx, _) = tokio::sync::broadcast::channel(256); - let (cleanup_tx, cleanup_rx) = mpsc::channel(CLEANUP_QUEUE_CAPACITY); + let (cleanup_tx, cleanup_rx) = mpsc::channel(cleanup_capacity); + let (control_cleanup_tx, control_cleanup_rx) = mpsc::channel(CONTROL_CLEANUP_CAPACITY); user_manager.set_cleanup_sender(cleanup_tx.clone()); user_manager.set_provider_manager(Arc::clone(provider_manager)); let socket_cleanup_tx = cleanup_tx.clone(); let socket_activity_tracker = SocketActivityTracker::new(); let capacity_notify = Arc::new(Notify::new()); + let shutdown_token = CancellationToken::new(); let mgr = Self { user_manager: Arc::clone(user_manager), provider_manager: Arc::clone(provider_manager), shared_stream_manager: Arc::clone(shared_stream_manager), event_manager: Arc::clone(event_manager), close_socket_signal_tx, - cleanup_sender: BackpressureSender::new(cleanup_tx, "cleanup", CLEANUP_QUEUE_CAPACITY), + cleanup_sender: BackpressureSender::new(cleanup_tx, "cleanup", cleanup_capacity), + control_cleanup_tx: control_cleanup_tx.clone(), socket_activity_tracker: socket_activity_tracker.clone(), capacity_notify: Arc::clone(&capacity_notify), stream_uid_counter: AtomicU32::new(1), history_writer: Arc::clone(&history_writer), + is_shutting_down: AtomicBool::new(false), + admission_gate: RwLock::new(()), + shutdown_token: shutdown_token.clone(), + worker_handles: std::sync::Mutex::new(Vec::new()), }; - Self::spawn_cleanup_worker( + let cleanup_handle = Self::spawn_cleanup_worker( cleanup_rx, + control_cleanup_rx, Arc::clone(user_manager), Arc::clone(provider_manager), Arc::clone(shared_stream_manager), Arc::clone(event_manager), Arc::clone(&capacity_notify), history_writer, + shutdown_token.clone(), ); - Self::spawn_socket_activity_worker(socket_activity_tracker, Arc::clone(user_manager), socket_cleanup_tx); + let socket_handle = Self::spawn_socket_activity_worker( + socket_activity_tracker, + Arc::clone(user_manager), + socket_cleanup_tx, + shutdown_token, + ); + mgr.worker_handles + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend([cleanup_handle, socket_handle]); mgr } /// Reload the history writer on config change. Shuts down the old writer first so - /// `recover_pending_files` in `build_history_writer` does not collide with an active writer. + /// `recover_pending_files` does not collide with an active writer. Recovery runs on the + /// blocking pool instead of a runtime worker so large pending-file sets cannot stall + /// live stream heartbeats. pub async fn reload_history_writer(&self, config: Option<&StreamHistoryConfig>) { let old_writer = self.history_writer.swap(None); if let Some(w) = old_writer { w.shutdown().await; } - let new_writer = build_history_writer(config); + let new_writer = build_history_writer_async(config).await; self.history_writer.store(new_writer); } @@ -624,7 +895,8 @@ impl ConnectionManager { activity_tracker: SocketActivityTracker, user_manager: Arc, cleanup_tx: mpsc::Sender, - ) { + shutdown_token: CancellationToken, + ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut expiry_queue: BinaryHeap> = BinaryHeap::new(); let mut expiry_index: HashMap = HashMap::new(); @@ -655,14 +927,18 @@ impl ConnectionManager { tokio::select! { biased; + () = shutdown_token.cancelled() => break, () = activity_tracker.notified() => {} () = tokio::time::sleep(Duration::from_secs(expires_at.saturating_sub(now))) => {} } } else { - activity_tracker.notified().await; + tokio::select! { + () = shutdown_token.cancelled() => break, + () = activity_tracker.notified() => {} + } } } - }); + }) } async fn drain_pending_socket_activity( @@ -762,15 +1038,18 @@ impl ConnectionManager { .collect(); } + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] fn spawn_cleanup_worker( mut rx: mpsc::Receiver, + mut control_rx: mpsc::Receiver, user_manager: Arc, provider_manager: Arc, shared_stream_manager: Arc, event_manager: Arc, capacity_notify: Arc, history_writer: Arc>, - ) { + shutdown_token: CancellationToken, + ) -> tokio::task::JoinHandle<()> { let deps = CleanupWorkerDeps { user_manager, provider_manager, @@ -780,12 +1059,37 @@ impl ConnectionManager { history_writer, }; tokio::spawn(async move { - while let Some(event) = rx.recv().await { + loop { + let event = tokio::select! { + biased; + () = shutdown_token.cancelled() => break, + control_event = control_rx.recv() => control_event, + event = rx.recv() => event, + }; + let Some(event) = event else { break }; match event { + CleanupEvent::ReleaseSharedSubscriber { addr, subscriber_id, request_id, owner } => { + deps.shared_stream_manager.release_subscriber(subscriber_id).await; + handle_release_stream( + &deps, + addr, + Some(subscriber_id.stream_uid()), + PROVIDER_END_NOT_SET, + 0, + None, + None, + request_id, + owner, + ) + .await; + notify_capacity(deps.capacity_notify.as_ref()); + } CleanupEvent::ReleaseConnection { addr } => { handle_release_connection(&deps, addr).await; } CleanupEvent::ReleaseStream { + request_id, + owner, addr, stream_uid, provider_end_reason, @@ -801,13 +1105,16 @@ impl ConnectionManager { reconnect_count, provider_error_class, provider_http_status, + request_id, + owner, ) .await; } CleanupEvent::ReleaseProviderHandle { handle } => { - handle_release_provider_handle(&deps, handle).await; + handle_release_provider_handle(&deps, handle); } CleanupEvent::ReleaseStreamAndProviderHandle { + request_id, addr, stream_uid, handle, @@ -825,26 +1132,40 @@ impl ConnectionManager { reconnect_count, provider_error_class, provider_http_status, + request_id, ) .await; } - CleanupEvent::UpdateDetailAndReleaseProvider { addr, video_type, handle } => { - handle_update_detail_and_release_provider(&deps, addr, video_type, handle).await; - } - CleanupEvent::UpdateDetailAndReleaseProviderConnection { addr, video_type } => { - handle_update_detail_and_release_provider_connection(&deps, addr, video_type).await; + CleanupEvent::UpdateDetailAndReleaseProvider { addr, stream_uid, video_type, handle } => { + handle_update_detail_and_release_provider(&deps, addr, video_type, handle, stream_uid).await; } CleanupEvent::AdaptiveSessionExpired { stream_info } => { handle_adaptive_session_expired(&deps, stream_info).await; } + CleanupEvent::ConfirmPlaybackLease { owner, request_id } => { + if let Some(request_id) = request_id { + deps.provider_manager.confirm_identified_playback_activity(&owner, request_id); + } else { + deps.provider_manager.confirm_playback_activity(&owner); + } + } + CleanupEvent::Defer(future) => { + future.await; + } } } debug!("Cleanup worker exiting"); - }); + }) } pub fn send_cleanup(&self, event: CleanupEvent) { self.cleanup_sender.enqueue(event); } + pub fn cleanup_tx(&self) -> mpsc::Sender { self.cleanup_sender.tx.clone() } + + /// Reserved control lane for cleanup permits that must never be starved by a + /// saturated body-cleanup queue (HLS origin I/O, shutdown). + pub fn control_cleanup_tx(&self) -> mpsc::Sender { self.control_cleanup_tx.clone() } + pub fn dropped_cleanup_events(&self) -> u64 { self.cleanup_sender.dropped_count() + self.user_manager.dropped_cleanup_events.load(Ordering::Relaxed) } @@ -886,6 +1207,10 @@ impl ConnectionManager { self.close_connection_with_reason(addr, DisconnectReason::ClientClosed) } + pub async fn block_stream_by_uid(&self, uid: u32, virtual_id: VirtualId, block_secs: u64) { + self.user_manager.block_user_for_stream_uid(uid, virtual_id, block_secs).await; + } + pub fn close_connection_with_reason(&self, addr: &SocketAddr, reason: DisconnectReason) -> bool { if let Err(e) = self.close_socket_signal_tx.send(CloseConnectionSignal::WithReason(*addr, reason)) { debug_if_enabled!( @@ -918,7 +1243,7 @@ impl ConnectionManager { /// called before invoking this method. pub async fn release_provider_deferred(&self, addr: &SocketAddr) { let addr_owned = *addr; - self.provider_manager.release_connection(&addr_owned).await; + self.provider_manager.release_connection(&addr_owned); self.shared_stream_manager.release_connection(&addr_owned, true).await; notify_capacity(self.capacity_notify.as_ref()); } @@ -934,7 +1259,7 @@ impl ConnectionManager { // Provider release and capacity notification are deferred via `release_provider_deferred`. for stream_info in &removed.removed_streams { if let Some(session_token) = stream_info.session_token.as_deref() { - self.provider_manager.clear_provider_reservation(session_token).await; + self.provider_manager.clear_provider_reservation(session_token); } } for username in &removed.disconnected_users { @@ -960,7 +1285,7 @@ impl ConnectionManager { } pub async fn release_provider_connection(&self, addr: &SocketAddr) { - self.provider_manager.release_connection(addr).await; + self.provider_manager.release_connection(addr); self.shared_stream_manager.release_connection(addr, false).await; notify_capacity(self.capacity_notify.as_ref()); } @@ -987,9 +1312,18 @@ impl ConnectionManager { } } - pub async fn release_provider_handle(&self, provider_handle: Option) { + pub fn release_provider_handle(&self, provider_handle: Option) { if let Some(handle) = provider_handle { - self.provider_manager.release_handle(&handle).await; + self.provider_manager.release_handle(&handle); + notify_capacity(self.capacity_notify.as_ref()); + } + } + + /// Releases a managed provider owner by value: its drop performs the synchronous + /// slot release, then the capacity waiters are notified. + pub fn release_managed_provider_handle(&self, provider_handle: Option) { + if provider_handle.is_some() { + drop(provider_handle); notify_capacity(self.capacity_notify.as_ref()); } } @@ -1023,26 +1357,54 @@ impl ConnectionManager { pub fn capacity_notified(&self) -> Arc { Arc::clone(&self.capacity_notify) } - /// Emit disconnect records for all still-active streams and flush the history writer. + #[inline] + pub fn is_shutting_down(&self) -> bool { self.is_shutting_down.load(Ordering::Acquire) } + + pub(crate) async fn begin_admission(&self) -> Option> { + if self.is_shutting_down() { + return None; + } + let guard = self.admission_gate.read().await; + (!self.is_shutting_down()).then_some(guard) + } + + /// Emit disconnect records for all still-active streams, unregister shared streams, + /// drain queued cleanups, and flush the history writer. /// Call once at graceful shutdown before dropping the `ConnectionManager`. pub async fn shutdown(&self) { - let active_streams = self.user_manager.get_all_active_streams().await; - for stream_info in active_streams { + self.is_shutting_down.store(true, Ordering::Release); + let _admission_closed = self.admission_gate.write().await; + self.shared_stream_manager.shutdown().await; + let active_streams = self.user_manager.drain_for_shutdown().await; + for stream_info in &active_streams { let qos = self.event_manager.read_meter_qos(stream_info.meter_uid).await; let bytes_sent = qos.map(|qos| qos.bytes_total); let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); emit_disconnect_record( &self.history_writer, - &stream_info, + stream_info, DisconnectReason::Shutdown, &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, None, None, ); + self.event_manager.unregister_meter_client(stream_info.uid).await; } + // This terminal transition does not depend on queue capacity or body-held + // cleanup permits. Later guard drops are idempotent against the emptied indices. + self.provider_manager.shutdown(); if let Some(w) = self.history_writer.load_full() { w.shutdown().await; } + // Stop and join the cleanup and socket workers so no background task outlives + // the manager. Active claims were released directly above; later guard drops are + // idempotent against the emptied indices. + self.shutdown_token.cancel(); + let handles = + std::mem::take(&mut *self.worker_handles.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); + for handle in handles { + let _ = handle.await; + } } pub async fn add_connection(&self, addr: &SocketAddr) { self.user_manager.add_connection(addr).await; } @@ -1066,9 +1428,81 @@ impl ConnectionManager { history_mode: ConnectionHistoryMode, ) -> Option { let uid = self.next_stream_uid(); + let mut registered = self.update_connection_with_uid(update, history_mode, uid, None).await; + // This compatibility API has no body to hand the claim to; the caller owns the + // registered stream, so disarm the rollback before returning its metadata. + registered.commit(); + registered.into_display_stream() + } + + pub async fn update_connection_with_uid( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + uid: u32, + provider_request_id: Option, + ) -> RegisteredPlaybackRequest { + self.update_connection_with_uid_impl(update, history_mode, uid, provider_request_id, false).await + } + + /// Registers a request whose enclosing shared-subscriber body already owns the + /// guaranteed terminal cleanup permit for this exact stream UID. + pub async fn update_connection_with_uid_using_shared_cleanup( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + capability: SharedCleanupCapability, + provider_request_id: Option, + ) -> RegisteredPlaybackRequest { + self.update_connection_with_uid_impl(update, history_mode, capability.stream_uid(), provider_request_id, true) + .await + } + + async fn update_connection_with_uid_impl( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + uid: u32, + provider_request_id: Option, + cleanup_owned_by_shared_subscriber: bool, + ) -> RegisteredPlaybackRequest { let username = update.username; let fingerprint = update.fingerprint; let track_direct_body_activity = uses_direct_body_idle_timeout(update.stream_channel); + let Some(_admission) = self.begin_admission().await else { + warn!("Connection manager is shutting down; rejecting connection registration for user {username}"); + return RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::CleanupReceiverClosed); + }; + // Admission: reserve a guaranteed cleanup right before any registration mutation, + // bounded so a saturated or stalled cleanup queue cannot hang request setup. + let rollback_permit = if cleanup_owned_by_shared_subscriber { + None + } else { + match tokio::time::timeout(CLEANUP_ADMISSION_TIMEOUT, self.cleanup_tx().reserve_owned()).await { + Ok(Ok(permit)) => Some(permit), + Ok(Err(_)) => { + warn!("Cleanup receiver closed; rejecting connection registration for user {username}"); + return RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::CleanupReceiverClosed); + } + Err(_) => { + warn!("Cleanup admission timed out; rejecting connection registration for user {username}"); + return RegisteredPlaybackRequest::rejected( + uid, + ConnectionRejectionReason::CleanupAdmissionTimeout, + ); + } + } + }; + // The cleanup owner is armed before the first mutation, so a cancellation during + // `update_connection`, meter registration or any later await still releases the claim. + let mut cleanup = OwnedRequestCleanup { + addr: fingerprint.addr, + request_uid: uid, + provider_request_id, + owner: update.session_token.map(Arc::::from), + permit: rollback_permit, + finished: false, + }; if let Some(stream_info) = self .user_manager .update_connection(ActiveUserConnectionParams { @@ -1101,11 +1535,12 @@ impl ConnectionManager { } self.event_manager .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); - Some(stream_info) + RegisteredPlaybackRequest::with_cleanup(uid, Some(stream_info), Some(cleanup)) } else { + // Registration failed: nothing was inserted, so the cleanup must not fire. + cleanup.disarm(); warn!("Failed to register connection for user {username} at {}; disconnecting client", fingerprint.addr); - let _ = self.close_connection_signal(&fingerprint.addr); - None + RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::RegistrationFailed) } } @@ -1118,6 +1553,12 @@ impl ConnectionManager { self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); } } + + pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) { + if let Some(stream_info) = self.user_manager.update_stream_detail_by_uid(uid, video_type).await { + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + } + } } /// Build a new `StreamHistoryWriter` from the given config, running file recovery first. @@ -1133,6 +1574,22 @@ fn build_history_writer(config: Option<&StreamHistoryConfig>) -> Option) -> Option> { + let cfg = config?; + if !cfg.stream_history_enabled { + return None; + } + let directory = cfg.stream_history_directory.clone(); + match tokio::task::spawn_blocking(move || recover_pending_files(&directory)).await { + Ok(Ok(())) => {} + Ok(Err(e)) => log::warn!("Stream history recovery failed: {e}"), + Err(join_err) => log::warn!("Stream history recovery task panicked: {join_err}"), + } + Some(Arc::new(StreamHistoryWriter::new(cfg))) +} + /// Determine the disconnect reason from the provider-end signal and the stream's current state. /// /// Priority: If `update_stream_detail` switched the stream to custom-video mode @@ -1165,10 +1622,47 @@ fn resolve_disconnect_reason(provider_end_reason: u8, stream_info: &StreamInfo) match provider_end_reason { PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, PROVIDER_END_ERROR => DisconnectReason::ProviderError, + PROVIDER_END_PREEMPTED => DisconnectReason::Preempted, _ => DisconnectReason::ClientClosed, } } +/// Resolves a disconnect reason from the provider-end signal alone, for the path where +/// the user stream claim is already gone and no `StreamInfo` is available to consult. +fn resolve_disconnect_reason_from_provider_end(provider_end_reason: u8) -> DisconnectReason { + match provider_end_reason { + PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, + PROVIDER_END_ERROR => DisconnectReason::ProviderError, + PROVIDER_END_PREEMPTED => DisconnectReason::Preempted, + _ => DisconnectReason::ClientClosed, + } +} + +/// Maps a disconnect reason onto the provider-lease outcome policy. +/// +/// Only a clean client-side end keeps a reconnect-capable lease alive; provider +/// failures, preemption, kicks and timeouts release capacity immediately. +fn playback_outcome_for_reason(reason: DisconnectReason) -> PlaybackRequestOutcome { + match reason { + DisconnectReason::ClientClosed + | DisconnectReason::DayRollover + | DisconnectReason::Cleanup + | DisconnectReason::Unknown => PlaybackRequestOutcome::ClientClosed, + DisconnectReason::Timeout => PlaybackRequestOutcome::TimedOut, + DisconnectReason::SessionExpired => PlaybackRequestOutcome::SessionExpired, + DisconnectReason::Shutdown => PlaybackRequestOutcome::ServerShutdown, + DisconnectReason::ClientKicked => PlaybackRequestOutcome::Kicked, + DisconnectReason::Preempted => PlaybackRequestOutcome::Preempted, + DisconnectReason::Provisioning + | DisconnectReason::UserConnectionsExhausted + | DisconnectReason::ProviderConnectionsExhausted => PlaybackRequestOutcome::FailedBeforeMedia, + DisconnectReason::ProviderError + | DisconnectReason::ProviderClosed + | DisconnectReason::ServerError + | DisconnectReason::IntermediateFailures(_) => PlaybackRequestOutcome::ProviderFailed, + } +} + fn emit_connect_record(writer: &ArcSwapOption, info: &StreamInfo) { let guard = writer.load(); let Some(w) = guard.as_ref() else { return }; @@ -1313,7 +1807,7 @@ mod tests { let event_manager = Arc::new(EventManager::new()); let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); - provider_manager.set_shared_stream_manager(Arc::clone(&shared_manager)); + provider_manager.set_shared_stream_manager(&shared_manager); let geo_ip = Arc::new(ArcSwapOption::::default()); let config = app_cfg.config.load(); @@ -1540,7 +2034,8 @@ mod tests { history_writer: Arc::clone(&manager.history_writer), }; - handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None).await; + handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None, None) + .await; assert!( manager @@ -1606,7 +2101,8 @@ mod tests { history_writer: Arc::clone(&manager.history_writer), }; - handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None).await; + handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None, None) + .await; assert!( manager @@ -1669,6 +2165,17 @@ mod tests { assert_eq!(reason, DisconnectReason::Preempted); } + #[test] + fn test_provider_preempted_atomic_maps_to_preempted() { + let info = make_stream_info("some_provider", "Some Channel"); + let reason = resolve_disconnect_reason(PROVIDER_END_PREEMPTED, &info); + assert_eq!(reason, DisconnectReason::Preempted); + + let reason_direct = resolve_disconnect_reason_from_provider_end(PROVIDER_END_PREEMPTED); + assert_eq!(reason_direct, DisconnectReason::Preempted); + assert_eq!(playback_outcome_for_reason(reason_direct), PlaybackRequestOutcome::Preempted); + } + #[test] fn test_channel_unavailable_with_eof_maps_to_provider_closed() { let info = make_stream_info("tuliprox", "channel_unavailable"); @@ -1756,4 +2263,537 @@ mod tests { Some(FailureStage::Streaming) ); } + + #[tokio::test] + async fn deferred_cleanup_executes_in_background_worker() { + let conn_manager = create_test_connection_manager(); + let ran = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let notify = Arc::new(Notify::new()); + let ran_clone = Arc::clone(&ran); + let notify_clone = Arc::clone(¬ify); + + conn_manager.send_cleanup(CleanupEvent::Defer(Box::pin(async move { + ran_clone.store(true, Ordering::SeqCst); + notify_clone.notify_one(); + }))); + + tokio::time::timeout(Duration::from_secs(1), notify.notified()).await.expect("deferred task must execute"); + assert!(ran.load(Ordering::SeqCst)); + } + + /// Dropping a just-registered request without handing it to a body must roll the + /// claim back, so a cancellation between registration and body construction cannot + /// leak a user claim. + #[tokio::test] + async fn registration_rollback_releases_claim_when_body_never_constructed() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56234".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 410, + title: "channel-410".intern(), + ..make_stream_info("provider_1", "channel-410").channel + }; + + manager.add_connection(&addr).await; + let registered = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "rollback-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + 1, + None, + ) + .await; + assert!(registered.display_stream.is_some(), "registration must succeed"); + + // Drop without committing: the rollback owner releases the claim. + drop(registered); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if manager.user_manager.active_streams().await.is_empty() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the rollback must release the claim"); + } + + /// The body cleanup transferred via `into_body_cleanup` releases the claim exactly + /// once with the real outcome, even under cleanup-queue pressure. + #[tokio::test] + async fn body_cleanup_finish_releases_claim_exactly_once() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56236".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 412, + title: "channel-412".intern(), + ..make_stream_info("provider_1", "channel-412").channel + }; + + manager.add_connection(&addr).await; + let mut registered = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "body-cleanup-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + 1, + None, + ) + .await; + assert!(registered.display_stream.is_some(), "registration must succeed"); + + let mut cleanup = registered.into_body_cleanup().expect("body cleanup present"); + cleanup.finish(None, PROVIDER_END_CLOSED, 0, None, None); + cleanup.finish(None, PROVIDER_END_CLOSED, 0, None, None); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if manager.user_manager.active_streams().await.is_empty() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the body cleanup must release the claim exactly once"); + } + + /// A saturated cleanup queue must reject registration with an admission-timeout reason + /// (rather than hanging), so the caller can surface a defined non-success response. + #[tokio::test(start_paused = true)] + async fn cleanup_admission_timeout_rejects_registration_with_reason() { + let manager = create_test_connection_manager(); + // Stall the cleanup worker on a never-resolving defer, then fill the channel so + // `reserve_owned()` blocks and the bounded admission deadline fires. + let pending = || CleanupEvent::Defer(Box::pin(std::future::pending::<()>())); + manager.send_cleanup(pending()); + for _ in 0..CLEANUP_QUEUE_CAPACITY { + manager.send_cleanup(pending()); + } + + let addr: SocketAddr = "127.0.0.1:56242".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 414, + title: "channel-414".intern(), + ..make_stream_info("provider_1", "channel-414").channel + }; + + manager.add_connection(&addr).await; + let manager_for_task = Arc::clone(&manager); + let registration = tokio::spawn(async move { + manager_for_task + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "admission-timeout-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + 1, + None, + ) + .await + }); + + // Let the task reach the blocked cleanup reservation, then fire the deadline. + tokio::task::yield_now().await; + tokio::task::yield_now().await; + tokio::time::advance(CLEANUP_ADMISSION_TIMEOUT + Duration::from_secs(1)).await; + let registered = registration.await.expect("registration task must not panic"); + + assert!(registered.display_stream.is_none(), "saturated cleanup admission must be rejected"); + assert_eq!(registered.rejection_reason(), Some(ConnectionRejectionReason::CleanupAdmissionTimeout)); + } + + #[tokio::test] + async fn control_cleanup_lane_precedes_a_ready_body_cleanup_backlog() { + let manager = create_test_connection_manager(); + let release_blocker = Arc::new(Notify::new()); + let (blocker_started_tx, blocker_started_rx) = tokio::sync::oneshot::channel(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + + manager.send_cleanup(CleanupEvent::Defer(Box::pin({ + let release_blocker = Arc::clone(&release_blocker); + let order = Arc::clone(&order); + async move { + let _ = blocker_started_tx.send(()); + release_blocker.notified().await; + order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('B'); + } + }))); + blocker_started_rx.await.expect("cleanup blocker must start"); + + manager.send_cleanup(CleanupEvent::Defer(Box::pin({ + let order = Arc::clone(&order); + async move { order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('N') } + }))); + manager + .control_cleanup_tx() + .send(CleanupEvent::Defer(Box::pin({ + let order = Arc::clone(&order); + async move { order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('C') } + }))) + .await + .expect("control cleanup receiver must be open"); + + release_blocker.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).len() == 3 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("both queued cleanup events must run"); + + assert_eq!( + *order.lock().unwrap_or_else(std::sync::PoisonError::into_inner), + vec!['B', 'C', 'N'], + "a ready control cleanup must not be starved by the body cleanup backlog" + ); + } + + /// A joined shared body already holds the terminal cleanup permit. Registration must + /// reuse that ownership instead of waiting for a second permit from the same full queue. + #[tokio::test] + async fn shared_admission_does_not_wait_for_second_permit_it_prevents() { + let manager = create_test_connection_manager(); + let cleanup_tx = manager.cleanup_tx(); + let available = cleanup_tx.capacity(); + let mut permits = Vec::with_capacity(available); + for _ in 0..available { + permits.push(cleanup_tx.clone().try_reserve_owned().expect("cleanup permit")); + } + + let addr: SocketAddr = "127.0.0.1:56243".parse().expect("test socket"); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 415, + title: "channel-415".intern(), + shared: true, + shared_joined_existing: Some(true), + ..make_stream_info("provider_1", "channel-415").channel + }; + + manager.add_connection(&addr).await; + let mut registered = tokio::time::timeout( + Duration::from_millis(100), + manager.update_connection_with_uid_using_shared_cleanup( + ConnectionParams { + meter_uid: 0, + username: "shared-admission-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + SharedCleanupCapability::new(tuliprox_core::model::SharedSubscriberId::from_stream_uid(1)), + None, + ), + ) + .await + .expect("externally owned cleanup must not reserve another permit"); + + assert!(registered.display_stream.is_some(), "shared registration must succeed"); + assert!(registered.into_body_cleanup().is_some(), "the body still tracks its terminal outcome"); + let _ = manager.user_manager.release_stream_request_by_uid(&addr, 1).await; + drop(permits); + } + + /// A cleanup whose user row is already gone must still finish the provider request, + /// because the provider identity travels with the cleanup owner independently. + #[tokio::test] + async fn cleanup_finishes_provider_request_when_user_row_is_gone() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56240".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 413, + title: "channel-413".intern(), + ..make_stream_info("provider_1", "channel-413").channel + }; + let input_name = "provider_1".intern(); + let owner = "session-owner-g2"; + + // Acquire a provider slot with an identified lease. + let handle = manager + .provider_manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + crate::ConnectionKind::Normal, + Some(owner), + ) + .expect("acquire provider slot"); + let request_id = handle.playback_request_id.expect("identified request id"); + assert!(manager.provider_manager.provider_lease_usage(&input_name).total() > 0); + + // Register a user claim carrying the same owner and provider request id. + manager.add_connection(&addr).await; + let mut registered = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "g2-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(owner), + }, + ConnectionHistoryMode::EmitConnect, + 1, + Some(request_id), + ) + .await; + assert!(registered.display_stream.is_some()); + + let cleanup = registered.into_body_cleanup().expect("cleanup present"); + + // Remove the user row first, so the cleanup worker hits the NotFound path. + let _ = manager.user_manager.release_stream_request_by_uid(&addr, 1).await; + + drop(cleanup); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if manager.provider_manager.provider_lease_usage(&input_name).total() == 0 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("provider request must finish even without a user row"); + + manager.provider_manager.release_handle(&handle); + } + + /// Shutdown releases every active stream so no claim or provider slot survives. + #[tokio::test] + async fn shutdown_releases_active_streams() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56235".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 411, + title: "channel-411".intern(), + ..make_stream_info("provider_1", "channel-411").channel + }; + + manager.add_connection(&addr).await; + let stream_info = manager + .update_connection(ConnectionParams { + meter_uid: 0, + username: "shutdown-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }) + .await; + assert!(stream_info.is_some(), "registration must succeed"); + assert!(!manager.user_manager.active_streams().await.is_empty()); + + manager.shutdown().await; + assert!(manager.user_manager.active_streams().await.is_empty(), "shutdown must release active streams"); + } + + #[tokio::test] + async fn shutdown_parallel_requests_on_distinct_sockets_drains_all_claims() { + let manager = create_test_connection_manager(); + let first_addr: SocketAddr = "127.0.0.1:56241".parse().unwrap(); + let second_addr: SocketAddr = "127.0.0.1:56242".parse().unwrap(); + let first_fingerprint = + tuliprox_core::model::Fingerprint::new("range-first".to_string(), "127.0.0.1".to_string(), first_addr); + let second_fingerprint = + tuliprox_core::model::Fingerprint::new("range-second".to_string(), "127.0.0.1".to_string(), second_addr); + let mut channel = make_stream_info("provider_1", "parallel-range").channel; + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + let owner = "shutdown-parallel-range"; + let input_name = "provider_1".intern(); + let handle = manager + .provider_manager + .acquire_connection_with_grace_for_session( + &input_name, + &first_addr, + false, + 0, + crate::ConnectionKind::Normal, + Some(owner), + ) + .expect("provider allocation"); + let request_id = handle.playback_request_id.expect("provider request id"); + + manager.add_connection(&first_addr).await; + manager.add_connection(&second_addr).await; + let mut first = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "shutdown-range-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &first_fingerprint, + provider: Arc::clone(&input_name), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(owner), + }, + ConnectionHistoryMode::EmitConnect, + 1, + Some(request_id), + ) + .await; + let mut second = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "shutdown-range-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &second_fingerprint, + provider: Arc::clone(&input_name), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(owner), + }, + ConnectionHistoryMode::EmitConnect, + 2, + None, + ) + .await; + assert!(first.display_stream.is_some()); + assert!(second.display_stream.is_some()); + assert_eq!(manager.user_manager.active_streams().await.len(), 1); + assert_eq!(manager.user_manager.playback_resource_counts().await.0, 2); + assert_eq!(manager.provider_manager.get_provider_connections_count(), 1); + + let first_cleanup = first.into_body_cleanup().expect("first cleanup"); + let second_cleanup = second.into_body_cleanup().expect("second cleanup"); + manager.shutdown().await; + + assert_eq!(manager.user_manager.playback_resource_counts().await, (0, 0)); + assert_eq!(manager.provider_manager.get_provider_connections_count(), 0); + assert_eq!(manager.provider_manager.provider_lease_usage(&input_name).total(), 0); + + drop(first_cleanup); + drop(second_cleanup); + drop(handle); + } + + #[tokio::test] + async fn shutdown_with_all_cleanup_permits_held_completes() { + let manager = create_test_connection_manager(); + let mut permits = Vec::with_capacity(CLEANUP_QUEUE_CAPACITY); + for _ in 0..CLEANUP_QUEUE_CAPACITY { + permits.push(manager.cleanup_tx().reserve_owned().await.expect("cleanup receiver open")); + } + assert_eq!(manager.cleanup_tx().capacity(), 0); + + tokio::time::timeout(Duration::from_secs(2), manager.shutdown()) + .await + .expect("shutdown must not require cleanup queue capacity"); + drop(permits); + } + + #[tokio::test] + async fn new_with_capacity_sizes_the_cleanup_queue() { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); + provider_manager.set_shared_stream_manager(&shared_manager); + let geo_ip = Arc::new(ArcSwapOption::::default()); + let config = app_cfg.config.load(); + let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); + + let manager = Arc::new(ConnectionManager::new_with_capacity( + &user_manager, + &provider_manager, + &shared_manager, + &event_manager, + None, + 2, + )); + assert_eq!(manager.cleanup_tx().capacity(), 2); + + // A capacity of zero is clamped to one so the cleanup queue is always usable. + let manager = Arc::new(ConnectionManager::new_with_capacity( + &user_manager, + &provider_manager, + &shared_manager, + &event_manager, + None, + 0, + )); + assert_eq!(manager.cleanup_tx().capacity(), 1); + } } diff --git a/backend/session/src/lib.rs b/backend/session/src/lib.rs index 299fc28ea..7d156c58b 100644 --- a/backend/session/src/lib.rs +++ b/backend/session/src/lib.rs @@ -26,6 +26,7 @@ pub mod event_manager; pub mod meter; pub mod meter_registry; pub mod provider_dns_manager; +pub mod provider_leases; pub mod provider_lineup_manager; pub mod qos_aggregation_manager; pub mod response_headers; @@ -36,5 +37,6 @@ pub mod streams; pub use self::{ active_provider_manager::*, active_user_manager::*, admission::*, admission_strategy::*, connection_manager::*, - event_manager::*, meter::*, meter_registry::*, provider_lineup_manager::*, stream::*, streams::*, + event_manager::*, meter::*, meter_registry::*, provider_leases::*, provider_lineup_manager::*, stream::*, + streams::*, }; diff --git a/backend/session/src/provider_leases.rs b/backend/session/src/provider_leases.rs new file mode 100644 index 000000000..01da17f10 --- /dev/null +++ b/backend/session/src/provider_leases.rs @@ -0,0 +1,954 @@ +//! Provider capacity slot leases. +//! +//! A lease is one logical provider slot owned by a playback, never by a transport +//! socket. Active allocations and idle reservations are two states of the same +//! lease. The provider manager serializes allocation counters, indices and lease +//! transitions when deciding capacity. +//! +//! Lifecycle: +//! +//! ```text +//! allocation acquired -> Starting (short startup deadline, blocks nobody) +//! confirmed media activity -> Active (expires on inactivity) +//! request finished cleanly + reconnect capable -> Idle (reconnect window) +//! never confirmed / failed / expired -> removed +//! ``` +//! +//! A lease only reserves capacity against other playbacks once real media activity +//! confirmed it *and* the endpoint configured a reconnect window. A manifest +//! response with HTTP 200 is not playback: treating it as one let abandoned starts +//! and manifest retries pile up reservations that outlived the streams they belonged +//! to and blocked unrelated clients behind the same reverse proxy. + +use log::debug; +use shared::utils::sanitize_sensitive_info; +use std::{ + collections::{BTreeSet, HashMap, HashSet}, + sync::Arc, + time::Duration, +}; +use tokio::time::Instant as TokioInstant; +use tuliprox_core::model::{ + PlaybackKind, PlaybackLeaseId, PlaybackRequestId, PlaybackRequestOutcome, ProviderBindingTag, +}; + +/// Hard upper bound for an unconfirmed lease. +/// +/// Provider open plus first media byte normally completes well inside this window. +/// Anything that never confirms is an abandoned start and must stop holding +/// capacity, independently of the configured reconnect TTL. +pub const STARTING_LEASE_TTL_SECS: u64 = 5; + +#[derive(Debug, Clone)] +pub enum ProviderLeaseState { + Starting { expires_at: TokioInstant }, + Active { request_id: PlaybackRequestId, expires_at: TokioInstant }, + Idle { expires_at: TokioInstant }, +} + +impl ProviderLeaseState { + #[inline] + pub fn is_active(&self) -> bool { matches!(self, Self::Active { .. }) } + + #[inline] + pub fn is_idle(&self) -> bool { matches!(self, Self::Idle { .. }) } + + /// True once real media activity was recorded. + #[inline] + pub fn is_confirmed(&self) -> bool { matches!(self, Self::Active { .. } | Self::Idle { .. }) } + + fn expires_at(&self) -> TokioInstant { + match self { + Self::Starting { expires_at } | Self::Active { expires_at, .. } | Self::Idle { expires_at } => *expires_at, + } + } + + pub fn as_log_state(&self) -> &'static str { + match self { + Self::Starting { .. } => "starting", + Self::Active { .. } => "active", + Self::Idle { .. } => "idle", + } + } +} + +#[derive(Debug, Clone)] +pub struct ProviderSlotLease { + pub id: PlaybackLeaseId, + /// Stable playback owner. Never a socket address. + pub owner: Arc, + pub provider_name: Arc, + pub kind: PlaybackKind, + pub state: ProviderLeaseState, + pub request_id: PlaybackRequestId, + /// Monotonic incarnation of this owner's provider binding. + pub binding_generation: u64, + /// Requests attached to the current binding incarnation. Parallel segment and + /// range requests remain valid until that incarnation is explicitly rebound. + request_ids: HashSet, + /// Reconnect window configured for this playback; `0` means the endpoint wants no + /// reservation, so the lease never blocks foreign capacity. + pub idle_ttl_secs: u64, + pub created_at: TokioInstant, + pub last_activity_at: TokioInstant, +} + +impl ProviderSlotLease { + /// True when this lease holds a slot back for its playback against others. + #[inline] + fn reserves_capacity(&self) -> bool { self.state.is_confirmed() && self.idle_ttl_secs > 0 } + + pub fn idle_ttl_remaining_ms(&self, now: TokioInstant) -> u64 { + self.state + .expires_at() + .checked_duration_since(now) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX) + } + + #[inline] + pub fn contains_request(&self, request_id: PlaybackRequestId) -> bool { self.request_ids.contains(&request_id) } + + /// True once the lease's deadline has passed. + /// + /// A running `Active` stream without a reconnect window is exempt: only an + /// explicit finish/release may end it, never wall-clock expiry. + #[inline] + fn is_expired(&self, now: TokioInstant) -> bool { + self.state.expires_at() <= now + && !matches!(self.state, ProviderLeaseState::Active { .. } if self.idle_ttl_secs == 0) + } +} + +/// Capacity-relevant view of one provider's lease table. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ProviderLeaseUsage { + pub starting: usize, + pub active: usize, + pub idle: usize, +} + +impl ProviderLeaseUsage { + /// Logical slots held by this provider's leases. + #[inline] + pub fn total(&self) -> usize { self.starting + self.active + self.idle } +} + +type ByOwner = HashMap, PlaybackLeaseId>; +type ByProvider = HashMap, HashSet>; + +/// Owner-keyed table of provider capacity slot leases. +/// +/// One owner is one logical playback, so repeated requests of the same playback +/// (manifest retries, segment fetches, range reconnects) share one slot instead of +/// stacking a reservation per attempt. +#[derive(Debug, Default)] +pub struct ProviderLeaseTable { + leases: HashMap, + by_owner: ByOwner, + by_provider: ByProvider, + // Exactly one deadline per lease; renewals replace entries rather than leaving + // stale heap entries that grow with the number of segment requests. + expirations: BTreeSet<(TokioInstant, PlaybackLeaseId)>, +} + +impl ProviderLeaseTable { + #[inline] + pub fn is_empty(&self) -> bool { self.leases.is_empty() } + + #[inline] + pub fn len(&self) -> usize { self.leases.len() } + + /// Removes every lease during terminal server shutdown. + pub fn clear(&mut self) { *self = Self::default(); } + + pub fn lease(&self, id: PlaybackLeaseId) -> Option<&ProviderSlotLease> { self.leases.get(&id) } + + pub fn lease_of_owner(&self, owner: &str) -> Option<&ProviderSlotLease> { + self.by_owner.get(owner).and_then(|id| self.leases.get(id)) + } + + fn index(by_owner: &mut ByOwner, by_provider: &mut ByProvider, lease: &ProviderSlotLease) { + by_owner.insert(Arc::clone(&lease.owner), lease.id); + by_provider.entry(Arc::clone(&lease.provider_name)).or_default().insert(lease.id); + } + + fn deindex(by_owner: &mut ByOwner, by_provider: &mut ByProvider, lease: &ProviderSlotLease) { + if by_owner.get(&lease.owner) == Some(&lease.id) { + by_owner.remove(&lease.owner); + } + if let Some(ids) = by_provider.get_mut(&lease.provider_name) { + ids.remove(&lease.id); + if ids.is_empty() { + by_provider.remove(&lease.provider_name); + } + } + } + + fn remove_lease(&mut self, id: PlaybackLeaseId) -> Option { + let lease = self.leases.remove(&id)?; + self.expirations.remove(&(lease.state.expires_at(), id)); + Self::deindex(&mut self.by_owner, &mut self.by_provider, &lease); + Some(lease) + } + + fn rebind_lease_provider( + by_owner: &mut ByOwner, + by_provider: &mut ByProvider, + lease: &mut ProviderSlotLease, + new_provider: &Arc, + ) { + if lease.provider_name != *new_provider { + Self::deindex(by_owner, by_provider, lease); + lease.provider_name = Arc::clone(new_provider); + Self::index(by_owner, by_provider, lease); + } + } + + fn startup_expiry(now: TokioInstant, idle_ttl_secs: u64) -> TokioInstant { + let ttl = if idle_ttl_secs == 0 { STARTING_LEASE_TTL_SECS } else { STARTING_LEASE_TTL_SECS.min(idle_ttl_secs) }; + now + Duration::from_secs(ttl) + } + + fn confirmed_expiry(now: TokioInstant, idle_ttl_secs: u64) -> TokioInstant { + now + Duration::from_secs(idle_ttl_secs.max(1)) + } + + /// Claims a slot for one playback at allocation time, before any media flows. + /// + /// An existing lease on the same provider attaches the request to its current + /// generation. Selecting a different provider starts a new generation and + /// invalidates delayed mutations from the preceding binding. + pub fn begin_owner( + &mut self, + owner: &str, + provider_name: &Arc, + kind: PlaybackKind, + request_id: PlaybackRequestId, + ) -> PlaybackLeaseId { + let now = TokioInstant::now(); + if let Some(id) = self.by_owner.get(owner).copied() { + if self.leases.get(&id).is_some_and(|lease| lease.is_expired(now)) { + self.remove_lease(id); + } + if let Some(lease) = self.leases.get_mut(&id) { + self.expirations.remove(&(lease.state.expires_at(), id)); + if lease.provider_name != *provider_name { + Self::rebind_lease_provider(&mut self.by_owner, &mut self.by_provider, lease, provider_name); + lease.binding_generation = lease.binding_generation.saturating_add(1); + lease.request_ids.clear(); + lease.created_at = now; + lease.idle_ttl_secs = 0; + lease.state = ProviderLeaseState::Starting { expires_at: Self::startup_expiry(now, 0) }; + } else if lease.state.is_idle() { + lease.binding_generation = lease.binding_generation.saturating_add(1); + lease.request_ids.clear(); + lease.created_at = now; + lease.state = + ProviderLeaseState::Starting { expires_at: Self::startup_expiry(now, lease.idle_ttl_secs) }; + } else if !lease.state.is_confirmed() { + lease.state = ProviderLeaseState::Starting { + expires_at: Self::startup_expiry(lease.created_at, lease.idle_ttl_secs), + }; + } + // The kind is authoritative from the first acquire of this playback and + // is never overwritten, so a compatibility renewal cannot downgrade a + // reconnect-capable lease to a one-shot live TS one. + lease.request_id = request_id; + lease.request_ids.insert(request_id); + lease.last_activity_at = now; + self.expirations.insert((lease.state.expires_at(), id)); + return id; + } + } + + let lease = ProviderSlotLease { + id: PlaybackLeaseId::next(), + owner: Arc::from(owner), + provider_name: Arc::clone(provider_name), + kind, + state: ProviderLeaseState::Starting { expires_at: Self::startup_expiry(now, 0) }, + request_id, + binding_generation: 1, + request_ids: HashSet::from([request_id]), + idle_ttl_secs: 0, + created_at: now, + last_activity_at: now, + }; + let id = lease.id; + self.expirations.insert((lease.state.expires_at(), id)); + Self::index(&mut self.by_owner, &mut self.by_provider, &lease); + self.leases.insert(id, lease); + id + } + + /// Renews the reconnect window of an existing request claim. + /// + /// A confirmed lease stays confirmed and only extends its window, so a follow-up + /// request of a running playback never downgrades it. An unconfirmed lease stays + /// bounded by [`STARTING_LEASE_TTL_SECS`] and keeps blocking nobody. + pub fn renew_identified_owner( + &mut self, + owner: &str, + provider_name: &Arc, + kind: PlaybackKind, + request_id: PlaybackRequestId, + idle_ttl_secs: u64, + ) -> Option { + if owner.is_empty() { + return None; + } + let now = TokioInstant::now(); + if let Some(id) = self.by_owner.get(owner).copied() { + let lease = self.leases.get_mut(&id)?; + if lease.is_expired(now) { + return None; + } + if lease.provider_name != *provider_name || !lease.contains_request(request_id) { + return None; + } + self.expirations.remove(&(lease.state.expires_at(), id)); + let was_confirmed = lease.state.is_confirmed(); + // The renewing endpoint may know the true media kind (VOD/Series/HLS via + // classification), so adopt it β€” but never downgrade a reconnect-capable + // lease, because compatibility refreshes default to one-shot live TS and + // must not strip an HLS/DASH/Catchup playback of its reconnect window. + if !lease.kind.is_reconnect_capable() { + lease.kind = kind; + } + lease.last_activity_at = now; + lease.idle_ttl_secs = idle_ttl_secs; + lease.state = if was_confirmed { + ProviderLeaseState::Active { + request_id: lease.request_id, + expires_at: Self::confirmed_expiry(now, idle_ttl_secs), + } + } else { + ProviderLeaseState::Starting { expires_at: Self::startup_expiry(lease.created_at, idle_ttl_secs) } + }; + self.expirations.insert((lease.state.expires_at(), id)); + return Some(id); + } + None + } + + /// Renews the current binding without manufacturing a request identity. + pub fn renew_current_owner( + &mut self, + owner: &str, + provider_name: &Arc, + kind: PlaybackKind, + idle_ttl_secs: u64, + ) -> Option { + let request_id = self.lease_of_owner(owner)?.request_id; + self.renew_identified_owner(owner, provider_name, kind, request_id, idle_ttl_secs) + } + + /// Starts a new binding incarnation, including when the provider name stays the same. + pub fn rebind_owner( + &mut self, + owner: &str, + provider_name: &Arc, + kind: PlaybackKind, + request_id: PlaybackRequestId, + ) -> PlaybackLeaseId { + self.release_owner(owner); + self.begin_owner(owner, provider_name, kind, request_id) + } + + /// Records confirmed media activity for a playback owner. + /// + /// Only a confirmed lease with a reconnect window may reserve provider capacity + /// against other playbacks. + pub fn confirm_owner(&mut self, owner: &str) -> Option { + let id = self.by_owner.get(owner).copied()?; + self.apply_confirmation(id, true) + } + + /// Confirms media activity without granting a reservation right, so the lease + /// never blocks foreign capacity. Used when the provider has no free slot for a + /// reconnect window but the media bytes are still delivered. + pub fn confirm_owner_without_reservation(&mut self, owner: &str) -> Option { + let id = self.by_owner.get(owner).copied()?; + self.apply_confirmation(id, false) + } + + /// Confirms real media activity for a specific request ID of a playback lease. + /// If the lease has moved to a newer request ID, the confirmation is rejected. + pub fn confirm_identified_owner(&mut self, owner: &str, request_id: PlaybackRequestId) -> Option { + let id = self.by_owner.get(owner).copied()?; + let lease = self.leases.get(&id)?; + if !lease.contains_request(request_id) { + return None; + } + self.apply_confirmation(id, true) + } + + /// Same as [`Self::confirm_identified_owner`] but records no reservation right. + pub fn confirm_identified_owner_without_reservation( + &mut self, + owner: &str, + request_id: PlaybackRequestId, + ) -> Option { + let id = self.by_owner.get(owner).copied()?; + let lease = self.leases.get(&id)?; + if !lease.contains_request(request_id) { + return None; + } + self.apply_confirmation(id, false) + } + + fn apply_confirmation(&mut self, id: PlaybackLeaseId, reserve: bool) -> Option { + let now = TokioInstant::now(); + let expired = self.leases.get(&id).is_some_and(|lease| lease.is_expired(now)); + if expired { + self.remove_lease(id); + return None; + } + let lease = self.leases.get_mut(&id)?; + self.expirations.remove(&(lease.state.expires_at(), id)); + lease.last_activity_at = now; + if !reserve { + lease.idle_ttl_secs = 0; + } + lease.state = ProviderLeaseState::Active { + request_id: lease.request_id, + expires_at: Self::confirmed_expiry(now, lease.idle_ttl_secs), + }; + self.expirations.insert((lease.state.expires_at(), id)); + Some(id) + } + + /// Ends one playback request and applies the outcome-specific lease policy. + /// + /// A confirmed, reconnect-capable playback with a configured window keeps its + /// slot as an idle lease; everything else is removed immediately, so an aborted + /// start never occupies provider capacity. + pub fn finish_owner( + &mut self, + owner: &str, + outcome: PlaybackRequestOutcome, + idle_ttl_secs: Option, + ) -> Option { + let id = self.by_owner.get(owner).copied()?; + let now = TokioInstant::now(); + let keep_idle = { + let lease = self.leases.get_mut(&id)?; + lease.last_activity_at = now; + let ttl = idle_ttl_secs.unwrap_or(lease.idle_ttl_secs); + let keep = lease.state.is_confirmed() + && outcome.keeps_idle_lease() + && lease.kind.is_reconnect_capable() + && ttl > 0; + if keep { + self.expirations.remove(&(lease.state.expires_at(), id)); + lease.idle_ttl_secs = ttl; + lease.state = ProviderLeaseState::Idle { expires_at: now + Duration::from_secs(ttl) }; + self.expirations.insert((lease.state.expires_at(), id)); + } + keep + }; + if keep_idle { + return None; + } + self.remove_lease(id) + } + + pub fn release(&mut self, id: PlaybackLeaseId) -> Option { self.remove_lease(id) } + + pub fn release_owner(&mut self, owner: &str) -> Option { + let id = self.by_owner.get(owner).copied()?; + self.remove_lease(id) + } + + /// Releases the lease only if it matches the exact provider binding tag. + /// + /// The tag binds a lease id and its incarnation generation, so a delayed clear + /// cannot delete a successor lease that was recreated on the same account and + /// happens to share generation `1`. There is deliberately no owner-wide fallback + /// here: callers that need an administrative owner-wide release must use + /// [`Self::release_owner`] explicitly. + pub fn release_matching_owner( + &mut self, + owner: &str, + provider_name: &Arc, + binding_tag: ProviderBindingTag, + ) -> Option { + let id = self.by_owner.get(owner).copied()?; + let lease = self.leases.get(&id)?; + if lease.provider_name != *provider_name + || lease.id != binding_tag.lease_id + || lease.binding_generation != binding_tag.generation + { + return None; + } + self.remove_lease(id) + } + + /// Ends one identified playback request claim and applies the outcome-specific lease policy + /// once the last attached request finishes. + pub fn finish_identified_owner( + &mut self, + owner: &str, + request_id: PlaybackRequestId, + outcome: PlaybackRequestOutcome, + idle_ttl_secs: Option, + ) -> Option { + let id = self.by_owner.get(owner).copied()?; + let now = TokioInstant::now(); + let keep_idle = { + let lease = self.leases.get_mut(&id)?; + if !lease.request_ids.remove(&request_id) { + return None; + } + if !lease.request_ids.is_empty() { + if lease.request_id == request_id { + if let Some(next) = lease.request_ids.iter().next().copied() { + lease.request_id = next; + } + } + return None; + } + lease.last_activity_at = now; + let ttl = idle_ttl_secs.unwrap_or(lease.idle_ttl_secs); + let keep = lease.state.is_confirmed() + && outcome.keeps_idle_lease() + && lease.kind.is_reconnect_capable() + && ttl > 0; + if keep { + self.expirations.remove(&(lease.state.expires_at(), id)); + lease.idle_ttl_secs = ttl; + lease.state = ProviderLeaseState::Idle { expires_at: now + Duration::from_secs(ttl) }; + self.expirations.insert((lease.state.expires_at(), id)); + } + keep + }; + if keep_idle { + None + } else { + self.remove_lease(id) + } + } + + /// Releases one request claim. A stale generation cannot clear its successor. + pub fn release_identified_owner( + &mut self, + owner: &str, + request_id: PlaybackRequestId, + ) -> Option { + let id = self.by_owner.get(owner).copied()?; + let lease = self.leases.get_mut(&id)?; + if !lease.request_ids.remove(&request_id) { + return None; + } + if let Some(next) = lease.request_ids.iter().next().copied() { + if lease.request_id == request_id { + lease.request_id = next; + } + return None; + } + self.remove_lease(id) + } + + /// Detaches one request claim from the owner's lease without terminating the lease. + pub fn detach_identified_request(&mut self, owner: &str, request_id: PlaybackRequestId) -> bool { + let Some(id) = self.by_owner.get(owner).copied() else { + return false; + }; + let Some(lease) = self.leases.get_mut(&id) else { + return false; + }; + if !lease.request_ids.remove(&request_id) { + return false; + } + if lease.request_id == request_id { + if let Some(next) = lease.request_ids.iter().next().copied() { + lease.request_id = next; + } else { + lease.request_id = PlaybackRequestId::default(); + } + } + true + } + + /// The provider this owner currently holds a lease on. + pub fn provider_for_owner(&self, owner: &str) -> Option> { + self.lease_of_owner(owner).map(|lease| Arc::clone(&lease.provider_name)) + } + + pub fn usage(&self, provider_name: &Arc) -> ProviderLeaseUsage { + let mut usage = ProviderLeaseUsage::default(); + let Some(ids) = self.by_provider.get(provider_name) else { + return usage; + }; + for id in ids { + let Some(lease) = self.leases.get(id) else { continue }; + match lease.state { + ProviderLeaseState::Starting { .. } => usage.starting += 1, + ProviderLeaseState::Active { .. } => usage.active += 1, + ProviderLeaseState::Idle { .. } => usage.idle += 1, + } + } + usage + } + + /// Confirmed leases that reserve a slot on this provider for somebody else. + /// + /// Unconfirmed `Starting` leases and leases without a reconnect window are + /// excluded: a manifest response alone is not playback. `counted_owners` already + /// hold a real allocation, so their lease is the same slot and must not be added + /// on top of the connection counter. + pub fn foreign_reserved_slots( + &self, + provider_name: &Arc, + session_owner: Option<&str>, + counted_owners: &HashSet>, + ) -> usize { + let Some(ids) = self.by_provider.get(provider_name) else { + return 0; + }; + ids.iter() + .filter_map(|id| self.leases.get(id)) + .filter(|lease| lease.reserves_capacity()) + .filter(|lease| session_owner != Some(lease.owner.as_ref()) && !counted_owners.contains(&lease.owner)) + .count() + } + + pub fn has_foreign_reserved_lease(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { + let Some(ids) = self.by_provider.get(provider_name) else { + return false; + }; + ids.iter() + .filter_map(|id| self.leases.get(id)) + .filter(|lease| lease.reserves_capacity()) + .any(|lease| session_owner != Some(lease.owner.as_ref())) + } + + /// Removes expired leases of any state. + pub fn prune(&mut self, now: TokioInstant) -> Vec { + if self.leases.is_empty() { + return Vec::new(); + } + let mut removed = Vec::new(); + while let Some((deadline, id)) = self.expirations.first().copied() { + if deadline > now { + break; + } + self.expirations.pop_first(); + // Active leases with idle_ttl_secs == 0 (VOD, Series, Live-TS) + // represent a running stream with no reconnect window configured. + // They must not be pruned by wall-clock time; only explicit + // finish_owner / remove_lease can end them. + let should_remove = !matches!( + self.leases.get(&id), + Some(lease) + if matches!(lease.state, ProviderLeaseState::Active { .. }) + && lease.idle_ttl_secs == 0 + ); + if should_remove { + if let Some(lease) = self.remove_lease(id) { + debug!( + "Provider lease expired: provider={} owner={} state={} kind={} lease_id={} request_id={}", + sanitize_sensitive_info(&lease.provider_name), + sanitize_sensitive_info(&lease.owner), + lease.state.as_log_state(), + lease.kind, + lease.id, + lease.request_id + ); + removed.push(lease); + } + } + } + removed + } + + pub fn iter(&self) -> impl Iterator { self.leases.values() } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn provider(name: &str) -> Arc { Arc::from(name) } + + #[tokio::test(start_paused = true)] + async fn repeated_begin_keeps_one_starting_lease() { + let mut table = ProviderLeaseTable::default(); + for index in 0..5 { + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, PlaybackRequestId::from_raw(index)); + } + assert_eq!(table.len(), 1, "manifest retries must not stack provider slots"); + assert_eq!(table.usage(&provider("A")), ProviderLeaseUsage { starting: 1, active: 0, idle: 0 }); + } + + #[tokio::test(start_paused = true)] + async fn stale_request_cannot_confirm_a_newer_playback_generation() { + let mut table = ProviderLeaseTable::default(); + let stale_request = PlaybackRequestId::from_raw(1); + let current_request = PlaybackRequestId::from_raw(2); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, stale_request); + table.rebind_owner("owner", &provider("A"), PlaybackKind::LiveHls, current_request); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, current_request, 15); + + assert!(table.confirm_identified_owner("owner", stale_request).is_none()); + assert_eq!(table.usage(&provider("A")), ProviderLeaseUsage { starting: 1, active: 0, idle: 0 }); + assert!(table.confirm_identified_owner("owner", current_request).is_some()); + assert_eq!(table.usage(&provider("A")), ProviderLeaseUsage { starting: 0, active: 1, idle: 0 }); + } + + #[tokio::test(start_paused = true)] + async fn parallel_attached_claims_can_both_confirm() { + let mut table = ProviderLeaseTable::default(); + let req1 = PlaybackRequestId::from_raw(1); + let req2 = PlaybackRequestId::from_raw(2); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, req1); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, req2); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, req2, 15); + + assert!(table.confirm_identified_owner("owner", req1).is_some()); + assert_eq!(table.usage(&provider("A")), ProviderLeaseUsage { starting: 0, active: 1, idle: 0 }); + assert!(table.confirm_identified_owner("owner", req2).is_some()); + assert_eq!(table.usage(&provider("A")), ProviderLeaseUsage { starting: 0, active: 1, idle: 0 }); + } + + #[tokio::test(start_paused = true)] + async fn unconfirmed_lease_expires_on_startup_deadline() { + let mut table = ProviderLeaseTable::default(); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, PlaybackRequestId::from_raw(1)); + table.renew_identified_owner( + "owner", + &provider("A"), + PlaybackKind::LiveHls, + PlaybackRequestId::from_raw(1), + 15, + ); + + tokio::time::advance(Duration::from_secs(STARTING_LEASE_TTL_SECS + 1)).await; + assert_eq!(table.prune(TokioInstant::now()).len(), 1); + assert!(table.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn starting_lease_never_blocks_other_playbacks() { + let mut table = ProviderLeaseTable::default(); + table.begin_owner("owner-a", &provider("A"), PlaybackKind::LiveHls, PlaybackRequestId::from_raw(1)); + table.renew_identified_owner( + "owner-a", + &provider("A"), + PlaybackKind::LiveHls, + PlaybackRequestId::from_raw(1), + 15, + ); + + assert_eq!(table.foreign_reserved_slots(&provider("A"), Some("owner-b"), &HashSet::new()), 0); + assert!(!table.has_foreign_reserved_lease(&provider("A"), Some("owner-b"))); + } + + #[tokio::test(start_paused = true)] + async fn confirmed_lease_blocks_until_its_window_expires() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner-a", &provider("A"), PlaybackKind::LiveHls, request); + table.renew_identified_owner("owner-a", &provider("A"), PlaybackKind::LiveHls, request, 15); + assert!(table.confirm_owner("owner-a").is_some()); + + assert_eq!(table.foreign_reserved_slots(&provider("A"), Some("owner-b"), &HashSet::new()), 1); + assert!(table.has_foreign_reserved_lease(&provider("A"), Some("owner-b"))); + // The owner's own live allocation is not counted twice. + let mut counted = HashSet::new(); + counted.insert(Arc::from("owner-a")); + assert_eq!(table.foreign_reserved_slots(&provider("A"), Some("owner-b"), &counted), 0); + + tokio::time::advance(Duration::from_secs(16)).await; + assert_eq!(table.prune(TokioInstant::now()).len(), 1); + assert!(!table.has_foreign_reserved_lease(&provider("A"), Some("owner-b"))); + } + + #[tokio::test(start_paused = true)] + async fn confirmed_lease_without_window_does_not_block() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner-a", &provider("A"), PlaybackKind::Vod, request); + assert!(table.confirm_owner("owner-a").is_some()); + assert_eq!(table.foreign_reserved_slots(&provider("A"), Some("owner-b"), &HashSet::new()), 0); + } + + #[tokio::test(start_paused = true)] + async fn confirmation_without_reservation_records_media_but_never_blocks() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner-a", &provider("A"), PlaybackKind::LiveHls, request); + table.renew_identified_owner("owner-a", &provider("A"), PlaybackKind::LiveHls, request, 15); + + assert!(table.confirm_identified_owner_without_reservation("owner-a", request).is_some()); + assert_eq!(table.usage(&provider("A")), ProviderLeaseUsage { starting: 0, active: 1, idle: 0 }); + assert_eq!( + table.foreign_reserved_slots(&provider("A"), Some("owner-b"), &HashSet::new()), + 0, + "media-without-reservation must not reserve capacity against other playbacks" + ); + } + + #[tokio::test(start_paused = true)] + async fn stale_clear_after_same_account_rebind_preserves_new_lease() { + let mut table = ProviderLeaseTable::default(); + let req1 = PlaybackRequestId::from_raw(1); + let req2 = PlaybackRequestId::from_raw(2); + + // First incarnation: confirmed then cleanly finished into an idle window. + let first_id = table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, req1); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, req1, 15); + table.confirm_owner("owner"); + table.finish_owner("owner", PlaybackRequestOutcome::Completed, Some(15)); + let first_generation = table.lease_of_owner("owner").expect("idle lease").binding_generation; + let first_tag = ProviderBindingTag::new(first_id, first_generation); + + // Aβ†’A rebind on the same account starts a new incarnation (generation increments). + let second_id = table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, req2); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, req2, 15); + let second_generation = table.lease_of_owner("owner").expect("rebound lease").binding_generation; + let second_tag = ProviderBindingTag::new(second_id, second_generation); + assert_ne!(first_generation, second_generation, "Aβ†’A rebind must advance the incarnation"); + assert_eq!(first_id, second_id, "Aβ†’A rebind reuses the lease id"); + + // A stale detach/clear with the old generation must not delete the successor lease. + assert!(table.release_matching_owner("owner", &provider("A"), first_tag).is_none()); + assert!(table.lease_of_owner("owner").is_some(), "stale clear must preserve the new lease"); + + // A clear with the captured generation removes exactly the current incarnation. + assert!(table.release_matching_owner("owner", &provider("A"), second_tag).is_some()); + assert!(table.lease_of_owner("owner").is_none()); + } + + #[tokio::test(start_paused = true)] + async fn stale_clear_after_remove_and_recreate_preserves_successor() { + let mut table = ProviderLeaseTable::default(); + let req1 = PlaybackRequestId::from_raw(1); + let req2 = PlaybackRequestId::from_raw(2); + + let first_id = table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, req1); + let first_tag = ProviderBindingTag::new(first_id, table.lease(first_id).expect("lease").binding_generation); + table.release_owner("owner"); + + let second_id = table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, req2); + let second_tag = ProviderBindingTag::new(second_id, table.lease(second_id).expect("lease").binding_generation); + + assert_ne!(first_id, second_id, "a recreated lease must have a fresh lease id"); + assert_eq!( + first_tag.generation, second_tag.generation, + "generation alone is not unique across remove/recreate" + ); + + // A stale clear with the old tag must not delete the successor lease. + assert!(table.release_matching_owner("owner", &provider("A"), first_tag).is_none()); + assert!(table.lease_of_owner("owner").is_some(), "stale clear must preserve the successor lease"); + + assert!(table.release_matching_owner("owner", &provider("A"), second_tag).is_some()); + assert!(table.lease_of_owner("owner").is_none()); + } + + #[tokio::test(start_paused = true)] + async fn unconfirmed_lease_is_dropped_when_request_ends() { + let mut table = ProviderLeaseTable::default(); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, PlaybackRequestId::from_raw(1)); + assert!(table.finish_owner("owner", PlaybackRequestOutcome::Completed, Some(15)).is_some()); + assert!(table.is_empty(), "a manifest-only start must not keep provider capacity"); + } + + #[tokio::test(start_paused = true)] + async fn failed_outcome_releases_even_confirmed_lease() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, request); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, request, 15); + table.confirm_owner("owner"); + assert!(table.finish_owner("owner", PlaybackRequestOutcome::ProviderFailed, Some(15)).is_some()); + assert!(table.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn timeout_and_expiry_release_confirmed_leases_immediately() { + for outcome in [PlaybackRequestOutcome::TimedOut, PlaybackRequestOutcome::SessionExpired] { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, request); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, request, 15); + table.confirm_owner("owner"); + + assert!(table.finish_owner("owner", outcome, Some(15)).is_some()); + assert!(table.is_empty()); + } + } + + #[tokio::test(start_paused = true)] + async fn live_ts_lease_never_becomes_idle() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveTs, request); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveTs, request, 15); + table.confirm_owner("owner"); + assert!(table.finish_owner("owner", PlaybackRequestOutcome::Completed, Some(15)).is_some()); + assert!(table.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn renew_moves_owner_to_new_provider_without_leaking_slot() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + let id = table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, request); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, request, 15); + table.confirm_owner("owner"); + table.finish_owner("owner", PlaybackRequestOutcome::ClientClosed, Some(15)); + assert_eq!(table.usage(&provider("A")).idle, 1); + + let renewed = table.renew_identified_owner("owner", &provider("B"), PlaybackKind::LiveHls, request, 15); + assert_eq!(renewed, None, "refresh cannot rebind an existing lease"); + let rebound = table.rebind_owner("owner", &provider("B"), PlaybackKind::LiveHls, request); + assert_ne!(rebound, id); + assert_eq!(table.usage(&provider("A")).total(), 0); + // Explicit rebind starts unconfirmed on the new provider. + assert_eq!(table.usage(&provider("B")).total(), 1); + assert_eq!(table.usage(&provider("B")).starting, 1); + assert_eq!(table.len(), 1); + } + + #[tokio::test(start_paused = true)] + async fn zero_ttl_disables_reservation_without_clearing_active_identity() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, request); + table.confirm_identified_owner("owner", request); + assert!(table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, request, 0).is_some()); + assert_eq!(table.usage(&provider("A")).active, 1); + assert_eq!(table.foreign_reserved_slots(&provider("A"), Some("other"), &HashSet::new()), 0); + } + + #[tokio::test(start_paused = true)] + async fn begin_owner_replaces_expired_lease() { + let mut table = ProviderLeaseTable::default(); + let first = table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, PlaybackRequestId::from_raw(1)); + tokio::time::advance(Duration::from_secs(STARTING_LEASE_TTL_SECS + 1)).await; + + let second = table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, PlaybackRequestId::from_raw(2)); + + assert_ne!(first, second, "an expired lease must not be reused"); + assert_eq!(table.len(), 1); + assert_eq!(table.usage(&provider("A")).starting, 1); + } + + #[tokio::test(start_paused = true)] + async fn renew_identified_owner_rejects_expired_lease() { + let mut table = ProviderLeaseTable::default(); + let request = PlaybackRequestId::from_raw(1); + table.begin_owner("owner", &provider("A"), PlaybackKind::LiveHls, request); + table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, request, 15); + tokio::time::advance(Duration::from_secs(STARTING_LEASE_TTL_SECS + 1)).await; + + assert!(table.renew_identified_owner("owner", &provider("A"), PlaybackKind::LiveHls, request, 15).is_none()); + assert_eq!(table.prune(TokioInstant::now()).len(), 1); + assert!(table.is_empty()); + } +} diff --git a/backend/session/src/provider_lineup_manager.rs b/backend/session/src/provider_lineup_manager.rs index 4a1ede032..73ef5d4a5 100644 --- a/backend/session/src/provider_lineup_manager.rs +++ b/backend/session/src/provider_lineup_manager.rs @@ -14,7 +14,6 @@ use std::{ Arc, }, }; -use tokio::sync::RwLock; use tuliprox_core::{ model::{ is_input_expired, ConfigInput, GracePeriodOptions, ProviderAllocation, ProviderConfig, @@ -62,12 +61,12 @@ macro_rules! gen_provider_search { } fn get_or_create_provider_connection( - provider_connections: &DashMap, Arc>>, + provider_connections: &DashMap, Arc>>, provider_name: &Arc, -) -> Arc> { +) -> Arc> { provider_connections .entry(provider_name.clone()) - .or_insert_with(|| Arc::new(RwLock::new(ProviderConfigConnection::default()))) + .or_insert_with(|| Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default()))) .clone() } @@ -131,14 +130,14 @@ impl ProviderLineup { } } - async fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { + fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { match self { - ProviderLineup::Single(lineup) => lineup.get_next(grace_period_timeout_secs).await, - ProviderLineup::Multi(lineup) => lineup.get_next(grace_period_timeout_secs).await, + ProviderLineup::Single(lineup) => lineup.get_next(grace_period_timeout_secs), + ProviderLineup::Multi(lineup) => lineup.get_next(grace_period_timeout_secs), } } - async fn acquire_excluding( + fn acquire_excluding( &self, with_grace: bool, grace_period_timeout_secs: u64, @@ -149,11 +148,11 @@ impl ProviderLineup { if excluded_providers.contains(&lineup.provider.name) { ProviderAllocation::Exhausted } else { - lineup.acquire(with_grace, grace_period_timeout_secs).await + lineup.acquire(with_grace, grace_period_timeout_secs) } } ProviderLineup::Multi(lineup) => { - lineup.acquire_excluding(with_grace, grace_period_timeout_secs, excluded_providers).await + lineup.acquire_excluding(with_grace, grace_period_timeout_secs, excluded_providers) } } } @@ -168,7 +167,7 @@ struct SingleProviderLineup { impl SingleProviderLineup { fn new( cfg: &ConfigInput, - connection: Arc>, + connection: Arc>, connection_change: &ProviderConnectionChangeCallback, ) -> Self { Self { @@ -176,18 +175,18 @@ impl SingleProviderLineup { } } - async fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { - self.provider.get_next(false, grace_period_timeout_secs).await + fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { + self.provider.get_next(false, grace_period_timeout_secs) } - async fn acquire(&self, with_grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { - self.provider.try_allocate(with_grace, grace_period_timeout_secs).await + fn acquire(&self, with_grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { + self.provider.try_allocate(with_grace, grace_period_timeout_secs) } #[cfg(test)] - async fn release(&self, provider_name: &Arc) { + fn release(&self, provider_name: &Arc) { if &self.provider.name == provider_name { - self.provider.release().await; + self.provider.release(); } } } @@ -225,7 +224,7 @@ struct MultiProviderLineup { impl MultiProviderLineup { pub fn new( cfg_input: &ConfigInput, - provider_connections: &DashMap, Arc>>, + provider_connections: &DashMap, Arc>>, connection_change: &ProviderConnectionChangeCallback, ) -> Self { let input_connection = get_or_create_provider_connection(provider_connections, &cfg_input.name); @@ -305,7 +304,7 @@ impl MultiProviderLineup { /// ProviderAllocation::GracePeriod(provider) => println!("Provider with grace period {}", provider.name), /// } /// ``` - async fn acquire_next_provider_from_group( + fn acquire_next_provider_from_group( priority_group: &ProviderPriorityGroup, grace: bool, grace_period_timeout_secs: u64, @@ -316,7 +315,7 @@ impl MultiProviderLineup { if excluded_providers.contains(&p.name) { return ProviderAllocation::Exhausted; } - let result = p.try_allocate(grace, grace_period_timeout_secs).await; + let result = p.try_allocate(grace, grace_period_timeout_secs); if !matches!(result, ProviderAllocation::Exhausted) { return result; } @@ -328,7 +327,7 @@ impl MultiProviderLineup { if excluded_providers.contains(&p.name) { continue; } - let result = p.try_allocate(true, grace_period_timeout_secs).await; + let result = p.try_allocate(true, grace_period_timeout_secs); if !matches!(result, ProviderAllocation::Exhausted) { return result; } @@ -342,7 +341,7 @@ impl MultiProviderLineup { loop { let p = &pg[idx]; if !excluded_providers.contains(&p.name) { - let result = p.try_allocate(grace, grace_period_timeout_secs).await; + let result = p.try_allocate(grace, grace_period_timeout_secs); if !matches!(result, ProviderAllocation::Exhausted) { index.store((idx + 1) % provider_count, Ordering::Release); return result; @@ -364,14 +363,14 @@ impl MultiProviderLineup { } // Used for redirect to cyclce through provider - async fn get_next_provider_from_group( + fn get_next_provider_from_group( priority_group: &ProviderPriorityGroup, grace: bool, grace_period_timeout_secs: u64, ) -> Option> { match priority_group { ProviderPriorityGroup::SingleProviderGroup(p) => { - return p.get_next(grace, grace_period_timeout_secs).await; + return p.get_next(grace, grace_period_timeout_secs); } ProviderPriorityGroup::MultiProviderGroup(index, pg) => { let provider_count = pg.len(); @@ -380,7 +379,7 @@ impl MultiProviderLineup { loop { let p = &pg[idx]; - let result = p.get_next(grace, grace_period_timeout_secs).await; + let result = p.get_next(grace, grace_period_timeout_secs); if result.is_some() { index.store((idx + 1) % provider_count, Ordering::Release); return result; @@ -426,11 +425,11 @@ impl MultiProviderLineup { /// } /// ``` #[cfg(test)] - async fn acquire(&self, with_grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { - self.acquire_excluding(with_grace, grace_period_timeout_secs, &std::collections::HashSet::new()).await + fn acquire(&self, with_grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { + self.acquire_excluding(with_grace, grace_period_timeout_secs, &std::collections::HashSet::new()) } - async fn acquire_excluding( + fn acquire_excluding( &self, with_grace: bool, grace_period_timeout_secs: u64, @@ -444,8 +443,7 @@ impl MultiProviderLineup { false, grace_period_timeout_secs, excluded_providers, - ) - .await; + ); if !matches!(allocation, ProviderAllocation::Exhausted) { return allocation; } @@ -462,8 +460,7 @@ impl MultiProviderLineup { true, grace_period_timeout_secs, excluded_providers, - ) - .await; + ); if !matches!(allocation, ProviderAllocation::Exhausted) { return allocation; } @@ -473,22 +470,18 @@ impl MultiProviderLineup { } // it intended to use with redirects to cycle through provider - async fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { + fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { // Prefer providers with available capacity (no grace allocations), // scanning priority groups from highest -> lowest. for priority_group in &self.providers { - if let Some(config) = - Self::get_next_provider_from_group(priority_group, false, grace_period_timeout_secs).await - { + if let Some(config) = Self::get_next_provider_from_group(priority_group, false, grace_period_timeout_secs) { return Some(config); } } // If no provider is available, allow grace. for priority_group in &self.providers { - if let Some(config) = - Self::get_next_provider_from_group(priority_group, true, grace_period_timeout_secs).await - { + if let Some(config) = Self::get_next_provider_from_group(priority_group, true, grace_period_timeout_secs) { return Some(config); } } @@ -497,19 +490,19 @@ impl MultiProviderLineup { } #[cfg(test)] - async fn release(&self, provider_name: &Arc) { + fn release(&self, provider_name: &Arc) { for g in &self.providers { match g { ProviderPriorityGroup::SingleProviderGroup(pc) => { if &pc.name == provider_name { - pc.release().await; + pc.release(); break; } } ProviderPriorityGroup::MultiProviderGroup(_, group) => { for pc in group { if &pc.name == provider_name { - pc.release().await; + pc.release(); return; } } @@ -518,17 +511,17 @@ impl MultiProviderLineup { } } - pub async fn get_total_connections(&self) -> usize { + pub fn get_total_connections(&self) -> usize { let mut total_connections = 0; for group in &self.providers { match group { ProviderPriorityGroup::SingleProviderGroup(provider) => { - total_connections += provider.get_current_connections().await; + total_connections += provider.get_current_connections(); } ProviderPriorityGroup::MultiProviderGroup(_, providers) => { for provider in providers { - total_connections += provider.get_current_connections().await; + total_connections += provider.get_current_connections(); } } } @@ -541,7 +534,7 @@ pub struct ProviderLineupManager { grace_period_millis: AtomicU64, grace_period_timeout_secs: AtomicU64, snapshot: Arc>, - provider_connections: DashMap, Arc>>, + provider_connections: DashMap, Arc>>, event_manager: Arc, /// Which priority group last served each input, so a fallback is reported /// on transition rather than once per allocation. @@ -560,7 +553,7 @@ impl ProviderLineupManager { grace_period_options: GracePeriodOptions, event_manager: &Arc, ) -> Self { - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineups = inputs.iter().map(|i| Self::create_lineup(i, &provider_connections, event_manager)).collect(); Self { grace_period_millis: AtomicU64::new(grace_period_options.period_millis), @@ -574,7 +567,7 @@ impl ProviderLineupManager { fn create_lineup( cfg_input: &ConfigInput, - provider_connections: &DashMap, Arc>>, + provider_connections: &DashMap, Arc>>, event_manager: &Arc, ) -> ProviderLineup { let event_manager = Arc::clone(event_manager); @@ -671,7 +664,7 @@ impl ProviderLineupManager { self.snapshot.store(Arc::new(LineupSnapshot { inputs: new_inputs, providers: new_lineups })); } - pub async fn reconcile_connections(&self, mut counts: HashMap, usize>) { + pub fn reconcile_connections(&self, mut counts: HashMap, usize>) { // Synchronize known providers from actual counts. // We take a snapshot of the keys and locks to avoid holding the DashMap's internal // shard locks while awaiting the RwLock of each provider. Holding both can lead to deadlocks. @@ -680,7 +673,10 @@ impl ProviderLineupManager { for (name, conn_lock) in snapshot { let count = counts.remove(&name).unwrap_or(0); - let mut conn = conn_lock.write().await; + let mut conn = match conn_lock.write() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; conn.current_connections = count; if count == 0 { conn.grace_started_at = None; @@ -694,9 +690,12 @@ impl ProviderLineupManager { let conn_lock = self .provider_connections .entry(name) - .or_insert_with(|| Arc::new(RwLock::new(ProviderConfigConnection::default()))) + .or_insert_with(|| Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default()))) .clone(); - let mut conn = conn_lock.write().await; + let mut conn = match conn_lock.write() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; conn.current_connections = count; } @@ -707,7 +706,10 @@ impl ProviderLineupManager { self.provider_connections.iter().map(|e| (e.key().clone(), Arc::clone(e.value()))).collect(); for (name, conn_lock) in snapshot { - let count = conn_lock.read().await.current_connections; + let count = match conn_lock.read() { + Ok(guard) => guard.current_connections, + Err(poisoned) => poisoned.into_inner().current_connections, + }; self.event_manager.send_event(EventMessage::ActiveProvider(name, count)); } } @@ -775,7 +777,7 @@ impl ProviderLineupManager { } /// Acquire exactly the requested provider account while respecting its configured limits. - pub async fn acquire_exact_connection_with_grace_override( + pub fn acquire_exact_connection_with_grace_override( &self, provider_name: &Arc, allow_grace: bool, @@ -785,7 +787,7 @@ impl ProviderLineupManager { let allocation = match Self::get_provider_config_by_name(provider_name, &snapshot.providers) { None => ProviderAllocation::Exhausted, Some((_lineup, config)) => { - config.try_allocate(with_grace, self.grace_period_timeout_secs.load(Ordering::Acquire)).await + config.try_allocate(with_grace, self.grace_period_timeout_secs.load(Ordering::Acquire)) } }; Self::log_allocation(&allocation); @@ -796,7 +798,7 @@ impl ProviderLineupManager { /// /// When `allow_grace` is `false`, the lineup will not allocate providers in `GracePeriod`, /// even if a global grace period is configured. - pub async fn acquire_connection_with_grace_override( + pub fn acquire_connection_with_grace_override( &self, input_name: &Arc, allow_grace: bool, @@ -806,10 +808,9 @@ impl ProviderLineupManager { allow_grace, &std::collections::HashSet::new(), ) - .await } - pub async fn acquire_connection_with_grace_override_excluding( + pub fn acquire_connection_with_grace_override_excluding( &self, input_name: &Arc, allow_grace: bool, @@ -820,19 +821,15 @@ impl ProviderLineupManager { let with_grace = allow_grace && self.grace_period_millis.load(Ordering::Acquire) > 0; let allocation = match lineup_opt { None => ProviderAllocation::Exhausted, // No Name matched, we don't have this provider - Some((lineup, _config)) => { - lineup - .acquire_excluding( - with_grace, - self.grace_period_timeout_secs.load(Ordering::Acquire), - excluded_providers, - ) - .await - } + Some((lineup, _config)) => lineup.acquire_excluding( + with_grace, + self.grace_period_timeout_secs.load(Ordering::Acquire), + excluded_providers, + ), }; if matches!(allocation, ProviderAllocation::Exhausted) { if let Some((lineup, _cfg)) = lineup_opt { - self.report_exhausted_pool(input_name, lineup).await; + self.report_exhausted_pool(input_name, lineup); } } else if let Some((lineup, _cfg)) = lineup_opt { if let Some(provider) = allocation.provider() { @@ -852,21 +849,21 @@ impl ProviderLineupManager { /// refused stream left nothing behind. It is built unconditionally now - /// this runs only on the exhausted path, which is already the slow one - /// and the debug line renders from the same data the event carries. - async fn report_exhausted_pool(&self, input_name: &Arc, lineup: &ProviderLineup) { + fn report_exhausted_pool(&self, input_name: &Arc, lineup: &ProviderLineup) { let mut entries: Vec = Vec::new(); match lineup { ProviderLineup::Single(single) => { - entries.push(Self::provider_snapshot_entry(&single.provider).await); + entries.push(Self::provider_snapshot_entry(&single.provider)); } ProviderLineup::Multi(multi) => { for group in &multi.providers { match group { ProviderPriorityGroup::SingleProviderGroup(cfg) => { - entries.push(Self::provider_snapshot_entry(cfg).await); + entries.push(Self::provider_snapshot_entry(cfg)); } ProviderPriorityGroup::MultiProviderGroup(_, cfgs) => { for cfg in cfgs { - entries.push(Self::provider_snapshot_entry(cfg).await); + entries.push(Self::provider_snapshot_entry(cfg)); } } } @@ -898,10 +895,10 @@ impl ProviderLineupManager { ))); } - async fn provider_snapshot_entry(cfg: &ProviderConfigWrapper) -> ProviderPoolEntry { + fn provider_snapshot_entry(cfg: &ProviderConfigWrapper) -> ProviderPoolEntry { ProviderPoolEntry { name: sanitize_sensitive_info(&cfg.name).into_owned().into(), - current_connections: cfg.get_current_connections().await, + current_connections: cfg.get_current_connections(), max_connections: cfg.max_connections(), expired: is_input_expired(cfg.exp_date()), } @@ -928,12 +925,12 @@ impl ProviderLineupManager { // This method is used for redirects to cycle through provider // - pub async fn get_next_provider(&self, input_name: &Arc) -> Option> { + pub fn get_next_provider(&self, input_name: &Arc) -> Option> { let snapshot = self.snapshot.load_full(); match Self::get_provider_config_by_name(input_name, &snapshot.providers) { None => None, Some((lineup, _config)) => { - let cfg = lineup.get_next(self.grace_period_timeout_secs.load(Ordering::Relaxed)).await; + let cfg = lineup.get_next(self.grace_period_timeout_secs.load(Ordering::Relaxed)); if log_enabled!(log::Level::Debug) { if let Some(ref c) = cfg { debug!("Using provider {}", c.name); @@ -955,7 +952,7 @@ impl ProviderLineupManager { .map_or_else(Vec::new, |(lineup, _)| lineup.provider_names()) } - pub async fn active_connections(&self) -> Option, usize>> { + pub fn active_connections(&self) -> Option, usize>> { let mut result = HashMap::, usize>::new(); let mut add_provider = |name: &Arc, count: usize| { if count > 0 { @@ -966,11 +963,11 @@ impl ProviderLineupManager { for lineup in &snapshot.providers { match lineup { ProviderLineup::Single(provider_lineup) => { - let connections = provider_lineup.provider.get_current_connections().await; + let connections = provider_lineup.provider.get_current_connections(); add_provider(&provider_lineup.provider.name, connections); } ProviderLineup::Multi(provider_lineup) => { - let connections = provider_lineup.get_total_connections().await; + let connections = provider_lineup.get_total_connections(); add_provider(&provider_lineup.name, connections); } } @@ -982,42 +979,42 @@ impl ProviderLineupManager { } } - pub async fn active_connection_count(&self) -> usize { + pub fn active_connection_count(&self) -> usize { let mut count = 0; let snapshot = self.snapshot.load_full(); for lineup in &snapshot.providers { match lineup { ProviderLineup::Single(provider_lineup) => { - count += provider_lineup.provider.get_current_connections().await; + count += provider_lineup.provider.get_current_connections(); } ProviderLineup::Multi(provider_lineup) => { - count += provider_lineup.get_total_connections().await; + count += provider_lineup.get_total_connections(); } } } count } - pub async fn provider_capacity(&self, provider_name: &Arc) -> Option<(usize, usize)> { + pub fn provider_capacity(&self, provider_name: &Arc) -> Option<(usize, usize)> { let snapshot = self.snapshot.load_full(); let (_, cfg) = Self::get_provider_config_by_name(provider_name, &snapshot.providers)?; - let current = cfg.get_current_connections().await; + let current = cfg.get_current_connections(); Some((current, cfg.max_connections())) } - pub async fn is_over_limit(&self, provider_name: &Arc) -> bool { + pub fn is_over_limit(&self, provider_name: &Arc) -> bool { let snapshot = self.snapshot.load_full(); if let Some((_, config)) = Self::get_provider_config_by_name(provider_name, &snapshot.providers) { - config.is_over_limit(self.grace_period_timeout_secs.load(Ordering::Relaxed)).await + config.is_over_limit(self.grace_period_timeout_secs.load(Ordering::Relaxed)) } else { false } } - pub async fn is_exhausted(&self, provider_name: &Arc) -> bool { + pub fn is_exhausted(&self, provider_name: &Arc) -> bool { let snapshot = self.snapshot.load_full(); if let Some((_, config)) = Self::get_provider_config_by_name(provider_name, &snapshot.providers) { - config.is_exhausted().await + config.is_exhausted() } else { false } @@ -1060,7 +1057,7 @@ mod tests { macro_rules! should_available { ($lineup:expr, $provider_id:expr, $grace_period_timeout_secs: expr) => { sleep(Duration::from_millis(200)).await; - match $lineup.acquire(true, $grace_period_timeout_secs).await { + match $lineup.acquire(true, $grace_period_timeout_secs) { ProviderAllocation::Exhausted => assert!(false, "Should available and not exhausted"), ProviderAllocation::Available(provider) => assert_eq!(provider.id, $provider_id), ProviderAllocation::GracePeriod(provider) => { @@ -1072,7 +1069,7 @@ mod tests { macro_rules! should_grace_period { ($lineup:expr, $provider_id:expr, $grace_period_timeout_secs: expr) => { sleep(Duration::from_millis(200)).await; - match $lineup.acquire(true, $grace_period_timeout_secs).await { + match $lineup.acquire(true, $grace_period_timeout_secs) { ProviderAllocation::Exhausted => assert!(false, "Should grace period and not exhausted"), ProviderAllocation::Available(provider) => { assert!(false, "Should grace period and not available: {}", provider.id) @@ -1085,7 +1082,7 @@ mod tests { macro_rules! should_exhausted { ($lineup:expr, $grace_period_timeout_secs: expr) => { sleep(Duration::from_millis(200)).await; - match $lineup.acquire(true, $grace_period_timeout_secs).await { + match $lineup.acquire(true, $grace_period_timeout_secs) { ProviderAllocation::Exhausted => {} ProviderAllocation::Available(provider) => { assert!(false, "Should exhausted and not available: {}", provider.id) @@ -1156,7 +1153,7 @@ mod tests { input.aliases = Some(vec![alias]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); // Create MultiProviderLineup with the provider and alias let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); @@ -1181,7 +1178,7 @@ mod tests { // Adding alias with different priority input.aliases = Some(vec![alias]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); // The alias has a higher priority, so the alias should be acquired first let rt = tokio::runtime::Runtime::new().unwrap(); @@ -1203,7 +1200,7 @@ mod tests { // Adding multiple aliases input.aliases = Some(vec![alias1, alias2]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1233,7 +1230,7 @@ mod tests { // Adding alias input.aliases = Some(vec![alias1, alias2]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1261,7 +1258,7 @@ mod tests { fn test_acquire_when_capacity_available() { let cfg = create_config_input(1, &"provider5_1".intern(), 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let connection = get_or_create_provider_connection(&provider_connections, &cfg.name); let lineup = SingleProviderLineup::new(&cfg, connection, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); @@ -1282,7 +1279,7 @@ mod tests { fn test_release_connection() { let cfg = create_config_input(1, &"provider7_1".intern(), 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let connection = get_or_create_provider_connection(&provider_connections, &cfg.name); let lineup = SingleProviderLineup::new(&cfg, connection, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); @@ -1292,10 +1289,10 @@ mod tests { should_available!(lineup, 1, 5); should_grace_period!(lineup, 1, 5); should_exhausted!(lineup, 5); - lineup.release(&"provider7_1".intern()).await; + lineup.release(&"provider7_1".intern()); should_grace_period!(lineup, 1, 5); - lineup.release(&"provider7_1".intern()).await; - lineup.release(&"provider7_1".intern()).await; + lineup.release(&"provider7_1".intern()); + lineup.release(&"provider7_1".intern()); should_available!(lineup, 1, 5); should_grace_period!(lineup, 1, 5); should_exhausted!(lineup, 5); @@ -1313,7 +1310,7 @@ mod tests { // Create MultiProviderLineup with the provider and alias let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&cfg1, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1329,9 +1326,9 @@ mod tests { should_grace_period!(lineup, 1, 5); should_grace_period!(lineup, 2, 5); - lineup.release(&"provider8_1".intern()).await; - lineup.release(&"alias_2".intern()).await; - lineup.release(&"provider8_1".intern()).await; + lineup.release(&"provider8_1".intern()); + lineup.release(&"alias_2".intern()); + lineup.release(&"provider8_1".intern()); should_available!(lineup, 1, 5); should_grace_period!(lineup, 1, 5); @@ -1346,7 +1343,7 @@ mod tests { async fn test_concurrent_acquire() { let cfg = create_config_input(1, &"provider9_1".intern(), 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap, Arc>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let connection = get_or_create_provider_connection(&provider_connections, &cfg.name); let lineup = Arc::new(SingleProviderLineup::new(&cfg, connection, &change_callback)); let barrier = Arc::new(Barrier::new(6)); @@ -1357,7 +1354,7 @@ mod tests { let barrier = Arc::clone(&barrier); tasks.spawn(async move { barrier.wait().await; - lineup.acquire(true, 5).await + lineup.acquire(true, 5) }); } barrier.wait().await; diff --git a/backend/session/src/stream.rs b/backend/session/src/stream.rs index 3e81444d6..0386e3c32 100644 --- a/backend/session/src/stream.rs +++ b/backend/session/src/stream.rs @@ -1,3 +1,4 @@ +use crate::{ManagedProviderHandle, SharedCleanupCapability}; use axum::http::StatusCode; use bytes::Bytes; use futures::stream::BoxStream; @@ -7,7 +8,7 @@ use shared::{ }; use std::{collections::HashMap, sync::Arc}; use tokio_util::sync::CancellationToken; -use tuliprox_core::model::{GracePeriodOptions, ProviderHandle, StreamError}; +use tuliprox_core::model::{GracePeriodOptions, StreamError}; use url::Url; pub type BoxedProviderStream = BoxStream<'static, Result>; @@ -20,6 +21,7 @@ pub struct ProviderStreamFactoryResponse { pub stream: BoxedProviderStream, pub info: ProviderStreamInfo, pub provider_session_headers: HashMap, + pub has_upstream_owner: bool, } /// Controls whether a provider stream preserves its origin representation or normalizes it to identity bytes. @@ -51,6 +53,11 @@ pub fn uses_direct_body_idle_timeout(stream_channel: &StreamChannel) -> bool { ) } +/// How long a direct VOD/series body may make no read progress before the stream is +/// considered dead. The socket-expiry allowance for those streams must not be shorter, +/// otherwise a player that pauses to drain its buffer is disconnected mid-playback. +pub const DIRECT_BODY_IDLE_TIMEOUT_SECS: u64 = 90; + type StreamUrl = Arc; type ProviderName = Arc; @@ -67,6 +74,7 @@ pub enum ProviderStreamState { } pub struct StreamDetails { + pub shared_subscriber_id: Option, pub stream: Option, pub stream_info: ProviderStreamInfo, pub provider_name: Option>, @@ -78,19 +86,20 @@ pub struct StreamDetails { pub provider_grace_active: bool, pub disable_provider_grace: bool, pub reconnect_flag: Option, - pub provider_handle: Option, + pub provider_handle: Option, pub content_representation: ProviderContentRepresentationMode, /// Set when the stream was admitted via a user-grace strategy. Carried through to /// `stream_grace_period` so remaining strategies can be evaluated if the grace fails. pub grace_resolution_context: Option, + pub custom_reason: Option, } -/// Manual Clone: stream cannot be cloned so we set it to None on the clone. -/// This is safe because `StreamDetails` is only cloned in contexts where the -/// stream has already been moved out (e.g., constructing grace params). +/// Manual Clone: stream and `provider_handle` cannot be duplicated so we set them to None on the clone. +/// This preserves singular handle ownership so grace snapshots do not duplicate releases. impl Clone for StreamDetails { fn clone(&self) -> Self { Self { + shared_subscriber_id: None, stream: None, stream_info: self.stream_info.clone(), provider_name: self.provider_name.clone(), @@ -102,9 +111,10 @@ impl Clone for StreamDetails { provider_grace_active: self.provider_grace_active, disable_provider_grace: self.disable_provider_grace, reconnect_flag: self.reconnect_flag.clone(), - provider_handle: self.provider_handle.clone(), + provider_handle: None, content_representation: self.content_representation, grace_resolution_context: self.grace_resolution_context.clone(), + custom_reason: self.custom_reason, } } } @@ -112,6 +122,7 @@ impl Clone for StreamDetails { impl StreamDetails { pub fn from_stream(stream: BoxedProviderStream, grace_period_options: GracePeriodOptions) -> Self { Self { + shared_subscriber_id: None, stream: Some(stream), stream_info: None, provider_name: None, @@ -126,6 +137,7 @@ impl StreamDetails { provider_handle: None, content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, + custom_reason: None, } } #[inline] @@ -146,7 +158,7 @@ impl StreamDetails { } pub struct StreamingStrategy { - pub provider_handle: Option, + pub provider_handle: Option, pub provider_stream_state: ProviderStreamState, pub input_headers: Option>, } diff --git a/backend/session/src/streams/mod.rs b/backend/session/src/streams/mod.rs index ffc980b7d..c48d1b2a0 100644 --- a/backend/session/src/streams/mod.rs +++ b/backend/session/src/streams/mod.rs @@ -1,4 +1,5 @@ pub mod buffered_stream; +pub mod provider_body_owner; pub mod shared_stream_manager; -pub use self::{buffered_stream::*, shared_stream_manager::*}; +pub use self::{buffered_stream::*, provider_body_owner::*, shared_stream_manager::*}; diff --git a/backend/session/src/streams/provider_body_owner.rs b/backend/session/src/streams/provider_body_owner.rs new file mode 100644 index 000000000..14e9130d1 --- /dev/null +++ b/backend/session/src/streams/provider_body_owner.rs @@ -0,0 +1,568 @@ +use crate::BoxedProviderStream; +use bytes::Bytes; +use futures::{ + stream::Stream, + task::{Context, Poll}, + StreamExt, +}; +use log::debug; +use std::{ + cmp::max, + future::Future, + pin::Pin, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + select, + sync::{ + mpsc::{channel, error::TrySendError, Sender}, + Semaphore, + }, + time::{sleep, Instant, Sleep}, +}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +use tuliprox_core::model::{ProviderCloseReason, StreamError}; + +pub const DEFAULT_DIRECT_BODY_CHANNEL_SIZE: usize = 2; +pub const DEFAULT_DIRECT_BODY_MAX_BYTES: usize = 256 * 1024; +pub const DEFAULT_DIRECT_BODY_IDLE_SECS: u64 = 90; + +pub const DEFAULT_BUFFERED_CHANNEL_SIZE: usize = 1024; +pub const DEFAULT_BUFFERED_MAX_BYTES: usize = 5 * 1024 * 1024; +pub const DEFAULT_BUFFERED_IDLE_SECS: u64 = 10; + +#[derive(Debug, Clone, Copy)] +pub struct ProviderBodyOwnerConfig { + pub channel_capacity: usize, + pub max_buffer_bytes: usize, + pub idle_timeout: Duration, +} + +impl ProviderBodyOwnerConfig { + pub fn for_direct_body() -> Self { + Self { + channel_capacity: DEFAULT_DIRECT_BODY_CHANNEL_SIZE, + max_buffer_bytes: DEFAULT_DIRECT_BODY_MAX_BYTES, + idle_timeout: Duration::from_secs(DEFAULT_DIRECT_BODY_IDLE_SECS), + } + } + + pub fn for_buffered(buffer_size: usize, max_buffer_bytes: usize) -> Self { + Self { + channel_capacity: max(buffer_size, DEFAULT_BUFFERED_CHANNEL_SIZE), + max_buffer_bytes: if max_buffer_bytes == 0 { DEFAULT_BUFFERED_MAX_BYTES } else { max_buffer_bytes }, + idle_timeout: Duration::from_secs(DEFAULT_BUFFERED_IDLE_SECS), + } + } +} + +/// An upstream body owner task that decouples upstream body consumption from downstream polling. +/// It observes cancellation independently of downstream `poll_next`, ensuring that upstream +/// connections close promptly upon preemption or client disconnect. +pub struct ProviderBodyOwner { + stream: ReceiverStream>, + close_cancelled: Pin>, + cancel_token: CancellationToken, + semaphore: Arc, + max_buffer_bytes: usize, +} + +impl ProviderBodyOwner { + pub fn new( + stream: BoxedProviderStream, + config: ProviderBodyOwnerConfig, + cancel_token: CancellationToken, + completion_token: Option, + close_reason: Option>, + ) -> Self { + let (tx, rx) = channel(config.channel_capacity); + let semaphore = Arc::new(Semaphore::new(config.max_buffer_bytes)); + + tokio::spawn(Self::run_owner_task( + tx, + stream, + cancel_token.clone(), + completion_token, + close_reason, + Arc::clone(&semaphore), + config, + )); + + Self { + stream: ReceiverStream::new(rx), + close_cancelled: Box::pin(cancel_token.clone().cancelled_owned()), + cancel_token, + semaphore, + max_buffer_bytes: config.max_buffer_bytes, + } + } + + async fn run_owner_task( + tx: Sender>, + mut stream: BoxedProviderStream, + cancel_token: CancellationToken, + completion_token: Option, + close_reason: Option>, + semaphore: Arc, + config: ProviderBodyOwnerConfig, + ) { + let idle_timeout = config.idle_timeout; + let idle = sleep(idle_timeout); + tokio::pin!(idle); + + while !cancel_token.is_cancelled() { + select! { + biased; + () = cancel_token.cancelled() => { + debug!("Provider body owner task observed cancellation signal"); + break; + } + () = &mut idle => { + debug!("Provider body owner task idle timeout expired ({idle_timeout:?})"); + if let Some(reason) = &close_reason { + let _ = reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + ProviderCloseReason::IdleTimeout as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + cancel_token.cancel(); + break; + } + chunk = stream.next() => { + idle.as_mut().reset(Instant::now() + idle_timeout); + match chunk { + Some(Ok(bytes)) => { + if !Self::forward_chunk( + &tx, + &cancel_token, + &mut idle, + &semaphore, + close_reason.as_ref(), + config.max_buffer_bytes, + bytes, + ) + .await + { + break; + } + } + Some(Err(err)) => { + if let Some(reason) = &close_reason { + let _ = reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + ProviderCloseReason::ProviderError as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + select! { + biased; + () = cancel_token.cancelled() => {}, + () = &mut idle => { + debug!("Provider body owner task idle timeout expired while delivering provider error ({idle_timeout:?})"); + if let Some(reason) = &close_reason { + let _ = reason.compare_exchange( + ProviderCloseReason::Unspecified as u8, + ProviderCloseReason::IdleTimeout as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + cancel_token.cancel(); + } + _ = tx.send(Err(err)) => {}, + } + break; + } + None => { + debug!("Provider body owner task completed cleanly on EOF"); + break; + } + } + } + } + } + + // Explicitly drop stream before signalling completion, releasing reqwest/hyper socket + drop(stream); + drop(tx); + + if let Some(completion) = completion_token { + completion.cancel(); + } + } + + /// Forwards one upstream chunk into the channel. + /// + /// The chunk is split into pieces no larger than `max_buffer_bytes`, and each + /// piece acquires its own semaphore permits before it is queued. This keeps the + /// value held in the channel within the configured accounting budget instead of + /// enqueueing an unbounded `Bytes` that only reserved a capped amount. + /// + /// Returns `false` when the owner task must terminate: cancellation, an idle + /// timeout while blocked, or a closed client channel. The close reason and the + /// cancellation token are updated in that case. + async fn forward_chunk( + tx: &Sender>, + cancel_token: &CancellationToken, + idle: &mut Pin<&mut Sleep>, + semaphore: &Arc, + close_reason: Option<&Arc>, + max_buffer_bytes: usize, + bytes: Bytes, + ) -> bool { + let piece_size = max_buffer_bytes.max(1); + let mut offset = 0; + while offset < bytes.len() { + let end = (offset + piece_size).min(bytes.len()); + let piece = bytes.slice(offset..end); + let permits = piece.len(); + if permits > 0 { + let acquired = select! { + biased; + () = cancel_token.cancelled() => None, + () = idle.as_mut() => { + Self::set_close_reason(close_reason, ProviderCloseReason::IdleTimeout); + cancel_token.cancel(); + return false; + } + permit = Arc::clone(semaphore).acquire_many_owned(u32::try_from(permits).unwrap_or(u32::MAX)) => permit.ok(), + }; + let Some(permit) = acquired else { + cancel_token.cancel(); + return false; + }; + permit.forget(); + } + let send_res = match tx.try_send(Ok(piece)) { + Ok(()) => Ok(()), + Err(TrySendError::Full(item)) => { + select! { + biased; + () = cancel_token.cancelled() => Err(()), + () = idle.as_mut() => { + Self::set_close_reason(close_reason, ProviderCloseReason::IdleTimeout); + cancel_token.cancel(); + return false; + } + res = tx.send(item) => res.map_err(|_| ()), + } + } + Err(TrySendError::Closed(_)) => Err(()), + }; + if send_res.is_err() { + if permits > 0 { + semaphore.add_permits(permits); + } + Self::set_close_reason(close_reason, ProviderCloseReason::ClientClosed); + cancel_token.cancel(); + return false; + } + offset = end; + } + true + } + + fn set_close_reason(close_reason: Option<&Arc>, reason: ProviderCloseReason) { + if let Some(slot) = close_reason { + let _ = slot.compare_exchange( + ProviderCloseReason::Unspecified as u8, + reason as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + } +} + +impl Stream for ProviderBodyOwner { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + match Pin::new(&mut this.stream).poll_next(cx) { + Poll::Ready(Some(Ok(bytes))) => { + this.semaphore.add_permits(bytes.len().min(this.max_buffer_bytes)); + Poll::Ready(Some(Ok(bytes))) + } + Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => { + if this.close_cancelled.as_mut().poll(cx).is_ready() { + Poll::Ready(None) + } else { + Poll::Pending + } + } + } + } +} + +impl Drop for ProviderBodyOwner { + fn drop(&mut self) { self.cancel_token.cancel(); } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::{Stream, StreamExt}; + use tokio::sync::oneshot; + + struct GatedDropProbeStream { + gate: oneshot::Receiver<()>, + dropped: Option>, + yielded: bool, + } + + impl Stream for GatedDropProbeStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.yielded { + return Poll::Pending; + } + match Pin::new(&mut self.gate).poll(cx) { + Poll::Ready(Ok(())) => { + self.yielded = true; + Poll::Ready(Some(Ok(Bytes::from_static(b"chunk")))) + } + Poll::Ready(Err(_)) => Poll::Ready(None), + Poll::Pending => Poll::Pending, + } + } + } + + impl Drop for GatedDropProbeStream { + fn drop(&mut self) { + if let Some(dropped) = self.dropped.take() { + let _ = dropped.send(()); + } + } + } + + #[tokio::test] + async fn unpolled_body_owner_terminates_upstream_when_cancelled() { + let (gate_tx, gate_rx) = oneshot::channel(); + let (dropped_tx, dropped_rx) = oneshot::channel(); + let cancel = CancellationToken::new(); + let completion = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(0)); + + let upstream = GatedDropProbeStream { gate: gate_rx, dropped: Some(dropped_tx), yielded: false }; + let owner = ProviderBodyOwner::new( + Box::pin(upstream), + ProviderBodyOwnerConfig::for_direct_body(), + cancel.clone(), + Some(completion.clone()), + Some(close_reason.clone()), + ); + + // Cancel the owner WITHOUT EVER POLLING IT + cancel.cancel(); + + // Completion must fire + tokio::time::timeout(Duration::from_secs(1), completion.cancelled()) + .await + .expect("completion token must be signalled when owner task terminates"); + + // Upstream stream must have been dropped + gate_tx.send(()).unwrap_err(); // gate_rx is dropped when upstream is dropped! + + tokio::time::timeout(Duration::from_secs(1), dropped_rx) + .await + .expect("upstream stream must be dropped without downstream polling") + .expect("drop notification delivered"); + + drop(owner); + } + + struct TwoChunksThenErrStream { + count: usize, + dropped: Option>, + } + impl Stream for TwoChunksThenErrStream { + type Item = Result; + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + match self.count.cmp(&2) { + std::cmp::Ordering::Less => { + self.count += 1; + Poll::Ready(Some(Ok(Bytes::from_static(b"chunk")))) + } + std::cmp::Ordering::Equal => { + self.count += 1; + Poll::Ready(Some(Err(StreamError::Stream("test error".to_owned())))) + } + std::cmp::Ordering::Greater => Poll::Pending, + } + } + } + impl Drop for TwoChunksThenErrStream { + fn drop(&mut self) { + if let Some(d) = self.dropped.take() { + let _ = d.send(()); + } + } + } + + #[tokio::test] + async fn error_delivery_does_not_block_cancellation_when_channel_full() { + let (dropped_tx, dropped_rx) = oneshot::channel(); + let cancel = CancellationToken::new(); + let completion = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(0)); + + let upstream = TwoChunksThenErrStream { count: 0, dropped: Some(dropped_tx) }; + let owner = ProviderBodyOwner::new( + Box::pin(upstream), + ProviderBodyOwnerConfig::for_direct_body(), + cancel.clone(), + Some(completion.clone()), + Some(close_reason.clone()), + ); + + tokio::time::sleep(Duration::from_millis(50)).await; + cancel.cancel(); + + tokio::time::timeout(Duration::from_millis(500), completion.cancelled()) + .await + .expect("completion token must be signalled when owner task terminates on error delivery"); + + tokio::time::timeout(Duration::from_millis(500), dropped_rx) + .await + .expect("upstream stream must be dropped") + .expect("drop notification delivered"); + + drop(owner); + } + + struct FixedChunksStream { + remaining: Vec, + } + impl Stream for FixedChunksStream { + type Item = Result; + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.remaining.is_empty() { + Poll::Pending + } else { + Poll::Ready(Some(Ok(self.remaining.remove(0)))) + } + } + } + + #[tokio::test] + async fn oversized_upstream_chunk_is_forwarded_in_bounded_pieces() { + let max = DEFAULT_DIRECT_BODY_MAX_BYTES; + let upstream = FixedChunksStream { remaining: vec![Bytes::from(vec![7u8; max + 10])] }; + let mut owner = ProviderBodyOwner::new( + Box::pin(upstream), + ProviderBodyOwnerConfig { + channel_capacity: 2, + max_buffer_bytes: max, + idle_timeout: Duration::from_secs(60), + }, + CancellationToken::new(), + None, + None, + ); + + let mut total = 0usize; + while total < max + 10 { + let piece = owner.next().await.expect("stream still open").expect("no stream error"); + assert!(piece.len() <= max, "forwarded piece must respect max_buffer_bytes"); + total += piece.len(); + } + assert_eq!(total, max + 10); + } + + struct EndlessChunkStream { + yielded: usize, + } + impl Stream for EndlessChunkStream { + type Item = Result; + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.yielded >= 4 { + Poll::Pending + } else { + self.yielded += 1; + Poll::Ready(Some(Ok(Bytes::from_static(b"x")))) + } + } + } + + #[tokio::test] + async fn idle_timeout_terminates_owner_blocked_on_full_channel() { + let cancel = CancellationToken::new(); + let completion = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(0)); + let owner = ProviderBodyOwner::new( + Box::pin(EndlessChunkStream { yielded: 0 }), + ProviderBodyOwnerConfig { + channel_capacity: 1, + max_buffer_bytes: 1024, + idle_timeout: Duration::from_millis(50), + }, + cancel.clone(), + Some(completion.clone()), + Some(close_reason.clone()), + ); + + // Never poll `owner`: the channel stays full and the owner blocks in its send path. + tokio::time::timeout(Duration::from_secs(2), completion.cancelled()) + .await + .expect("owner must terminate after the idle timeout"); + assert_eq!(close_reason.load(Ordering::Acquire), ProviderCloseReason::IdleTimeout as u8); + + drop(owner); + } + + struct SingleChunkThenErrStream { + chunk_sent: bool, + } + impl Stream for SingleChunkThenErrStream { + type Item = Result; + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.chunk_sent { + Poll::Ready(Some(Err(StreamError::Stream("test error".to_owned())))) + } else { + self.chunk_sent = true; + Poll::Ready(Some(Ok(Bytes::from_static(b"chunk")))) + } + } + } + + #[tokio::test] + async fn idle_timeout_terminates_owner_blocked_delivering_provider_error() { + let cancel = CancellationToken::new(); + let completion = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(0)); + let owner = ProviderBodyOwner::new( + Box::pin(SingleChunkThenErrStream { chunk_sent: false }), + ProviderBodyOwnerConfig { + channel_capacity: 1, + max_buffer_bytes: 1024, + idle_timeout: Duration::from_millis(50), + }, + cancel.clone(), + Some(completion.clone()), + Some(close_reason.clone()), + ); + + // Never poll `owner`: the first chunk fills the channel, so the provider + // error send blocks. The idle timer must still terminate the owner task. + // The provider error was recorded before the send, so it stays the close + // reason (existing precedence); the timeout only unblocks termination. + tokio::time::timeout(Duration::from_secs(2), completion.cancelled()) + .await + .expect("owner must terminate after the idle timeout while delivering the provider error"); + assert_eq!(close_reason.load(Ordering::Acquire), ProviderCloseReason::ProviderError as u8); + + drop(owner); + } +} diff --git a/backend/session/src/streams/shared_stream_manager.rs b/backend/session/src/streams/shared_stream_manager.rs index c3c20bab7..de65b664e 100644 --- a/backend/session/src/streams/shared_stream_manager.rs +++ b/backend/session/src/streams/shared_stream_manager.rs @@ -1,28 +1,32 @@ -use crate::{streams::buffered_stream::CHANNEL_SIZE, ActiveProviderManager, BoxedProviderStream, ConnectionManager}; +use crate::{ + streams::buffered_stream::CHANNEL_SIZE, ActiveProviderManager, BoxedProviderStream, CleanupEvent, + ConnectionManager, ConnectionRejectionReason, ManagedProviderHandle, SharedCleanupCapability, +}; use bytes::Bytes; -use futures::{stream::BoxStream, Stream, StreamExt}; +use futures::{Stream, StreamExt}; use log::{debug, warn}; use shared::utils::sanitize_sensitive_info; use std::{ collections::{HashMap, VecDeque}, fmt, fmt::{Debug, Formatter}, + future::Future, net::SocketAddr, pin::Pin, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicUsize, Ordering}, Arc, }, task::{Context, Poll}, }; use tokio::{ - sync::{mpsc, mpsc::Sender, Mutex, Notify, RwLock}, - time::{sleep, Duration, Instant}, + sync::{mpsc, mpsc::Sender, oneshot, Mutex, Notify, RwLock, Semaphore}, + time::{sleep, timeout, Duration, Instant, Sleep}, }; use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::CancellationToken; use tuliprox_core::{ - model::{AppConfig, Config, ProviderHandle, StreamError}, + model::{AllocationId, AppConfig, Config, ProviderHandle, SharedSubscriberId, StreamError}, utils::{debug_if_enabled, network::request::STREAM_IDLE_TIMEOUT, trace_if_enabled}, }; @@ -32,9 +36,147 @@ const MIN_BURST_BUFFER_CHUNKS: usize = 2; const MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES: usize = 188; const SHARED_BURST_BYTES_PER_BUFFER_SLOT: usize = 12 * 1024; const DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS: u64 = 300; +const SHARED_CLEANUP_ADMISSION_TIMEOUT: Duration = Duration::from_secs(5); +// Upper bound on chunks pulled into a subscriber's live batch per read. The burst ring +// is byte-bounded already; this caps how many of its entries are cloned into the +// per-subscriber vector at once so a long backlog cannot balloon that vector. +const MAX_SUBSCRIBER_LIVE_BATCH_CHUNKS: usize = 64; + +pub struct PendingSharedSubscriberCleanup { + permit: Option>, + subscriber_id: SharedSubscriberId, + addr: SocketAddr, + request_id: Option, + owner: Option>, +} + +pub struct PendingSharedMeterRegistration { + manager: Arc, + stream_url: Arc, + meter_uid: u32, + armed: bool, +} + +impl PendingSharedMeterRegistration { + pub fn commit(mut self) { self.armed = false; } +} + +impl Drop for PendingSharedMeterRegistration { + fn drop(&mut self) { + if self.armed { + self.manager.remove_meter_uid_if_pending(&self.stream_url, self.meter_uid); + } + } +} + +/// Tracks a shared-stream meter reservation. `owner` is `None` while the meter is +/// only reserved; it becomes `Some(allocation_id)` once a live shared origin adopts it. +/// Pending rollbacks may only remove an entry that no origin has adopted yet. +struct SharedMeterEntry { + uid: u32, + owner: Option, +} + +/// RAII guard for a join that committed its registry entry but has not yet registered +/// its subscriber. It decrements the origin's pending-join counter exactly once, either +/// on successful registration (`commit`) or when the join future is dropped. +struct PendingJoinGuard { + state: Arc, + armed: bool, +} + +impl PendingJoinGuard { + fn new(state: &Arc) -> Self { + state.increment_pending_joins(); + Self { state: Arc::clone(state), armed: true } + } + + fn commit(mut self) { + self.armed = false; + self.state.decrement_pending_joins(); + } +} + +impl Drop for PendingJoinGuard { + fn drop(&mut self) { + if self.armed { + self.state.decrement_pending_joins(); + } + } +} + +impl PendingSharedSubscriberCleanup { + fn new( + permit: tokio::sync::mpsc::OwnedPermit, + subscriber_id: SharedSubscriberId, + addr: SocketAddr, + ) -> Self { + Self { permit: Some(permit), subscriber_id, addr, request_id: None, owner: None } + } + + pub fn set_provider_request_identity(&mut self, owner: &str, request_id: tuliprox_core::model::PlaybackRequestId) { + self.owner = Some(Arc::from(owner)); + self.request_id = Some(request_id); + } + + /// Produces the cleanup-ownership proof for this subscriber. The permit held by + /// this guard is the source of the capability. + pub fn capability(&self) -> SharedCleanupCapability { SharedCleanupCapability::new(self.subscriber_id) } + + fn take_cleanup( + &mut self, + ) -> ( + Option>, + Option, + Option>, + ) { + (self.permit.take(), self.request_id.take(), self.owner.take()) + } + + pub fn disarm(&mut self) { self.permit = None; } +} + +impl Drop for PendingSharedSubscriberCleanup { + fn drop(&mut self) { + if let Some(permit) = self.permit.take() { + permit.send(CleanupEvent::ReleaseSharedSubscriber { + addr: self.addr, + subscriber_id: self.subscriber_id, + request_id: self.request_id, + owner: self.owner.take(), + }); + } + } +} struct ReceiverStreamWrapper { stream: S, + start: Option>, + subscriber_id: SharedSubscriberId, + addr: SocketAddr, + /// Guaranteed cleanup right reserved before subscriber registration, so the + /// release below is never dropped by cleanup-queue pressure. + permit: Option>, + request_id: Option, + owner: Option>, + deadline: Option>>, + released: bool, +} + +impl ReceiverStreamWrapper { + fn release(&mut self) { + if !self.released { + self.released = true; + if let Some(permit) = self.permit.take() { + permit.send(CleanupEvent::ReleaseSharedSubscriber { + addr: self.addr, + subscriber_id: self.subscriber_id, + request_id: self.request_id, + owner: self.owner.take(), + }); + } + } + } } impl Stream for ReceiverStreamWrapper @@ -44,6 +186,13 @@ where type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.deadline.as_mut().is_some_and(|deadline| deadline.as_mut().poll(cx).is_ready()) { + self.release(); + return Poll::Ready(None); + } + if let Some(start) = self.start.take() { + let _ = start.send(()); + } match Pin::new(&mut self.stream).poll_next(cx) { Poll::Ready(Some(bytes)) => Poll::Ready(Some(Ok(bytes))), Poll::Ready(None) => Poll::Ready(None), @@ -62,23 +211,18 @@ fn resolve_min_burst_buffer_bytes(config: &Config) -> usize { .max(1) } -fn convert_stream(stream: BoxStream) -> BoxStream> { - ReceiverStreamWrapper { stream }.boxed() +impl Drop for ReceiverStreamWrapper { + fn drop(&mut self) { self.release(); } } -type SubscriberId = SocketAddr; +type SubscriberId = SharedSubscriberId; #[derive(Clone, Debug)] struct SharedSubscriber { - id: u64, + addr: SocketAddr, cancel_token: CancellationToken, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct SharedSubscriberOwner { - id: u64, -} - struct BufferedChunk { sequence: u64, bytes: Bytes, @@ -123,15 +267,19 @@ impl BurstBuffer { (self.buffer.iter().map(|chunk| chunk.bytes.clone()).collect::>(), self.next_sequence) } - pub fn read_from_into(&self, next_sequence: u64, chunks: &mut Vec) -> BurstRead { + pub fn read_from_into(&self, next_sequence: u64, chunks: &mut Vec, max_chunks: usize) -> BurstRead { chunks.clear(); let earliest_sequence = self.buffer.front().map_or(self.next_sequence, |chunk| chunk.sequence); let start_sequence = next_sequence.max(earliest_sequence); let skipped = start_sequence.saturating_sub(next_sequence); let start_index = self.start_index_for_sequence(start_sequence); - chunks.extend(self.buffer.range(start_index..).map(|chunk| chunk.bytes.clone())); + let mut read_next_sequence = start_sequence; + for chunk in self.buffer.range(start_index..).take(max_chunks) { + chunks.push(chunk.bytes.clone()); + read_next_sequence = chunk.sequence.saturating_add(1); + } - BurstRead { next_sequence: self.next_sequence, skipped } + BurstRead { next_sequence: read_next_sequence, skipped } } pub fn push(&mut self, packet: Bytes) { @@ -172,41 +320,92 @@ impl BurstBuffer { } } -async fn send_burst_buffer( - start_buffer: &[Bytes], - client_tx: &Sender, - cancellation_token: &CancellationToken, -) -> usize { - let mut sent = 0_usize; - for buf in start_buffer { - if cancellation_token.is_cancelled() { - return sent; - } - if !send_client_chunk(client_tx, buf.clone(), cancellation_token).await { - debug!("Failed sending burst-buffer chunk to client"); - return sent; - } - sent = sent.saturating_add(1); - } - sent +/// Terminal outcome of trying to deliver one chunk to a subscriber. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SendOutcome { + Sent, + Cancelled, + Closed, + TimedOut, } -async fn send_client_chunk(client_tx: &Sender, data: Bytes, cancellation_token: &CancellationToken) -> bool { - if cancellation_token.is_cancelled() { - return false; - } - - let data = match client_tx.try_send(data) { - Ok(()) => return true, - Err(mpsc::error::TrySendError::Closed(_)) => return false, - Err(mpsc::error::TrySendError::Full(data)) => data, - }; +/// A queue entry that releases its byte-budget permit when the client consumes (drops) it. +struct BudgetedChunk { + bytes: Bytes, + _permit: tokio::sync::OwnedSemaphorePermit, +} +async fn reserve_byte_permit( + byte_budget: &Arc, + len: usize, + cancellation_token: &CancellationToken, + progress_deadline: Instant, +) -> Result { + let permits = u32::try_from(len).unwrap_or(u32::MAX); tokio::select! { biased; + () = cancellation_token.cancelled() => Err(SendOutcome::Cancelled), + result = tokio::time::timeout_at(progress_deadline, byte_budget.clone().acquire_many_owned(permits)) => { + match result { + Ok(Ok(permit)) => Ok(permit), + Ok(Err(_)) => Err(SendOutcome::Closed), + Err(_) => Err(SendOutcome::TimedOut), + } + } + } +} - () = cancellation_token.cancelled() => false, - result = client_tx.send(data) => result.is_ok(), +async fn send_burst_buffer( + start_buffer: &[Bytes], + client_tx: &Sender, + cancellation_token: &CancellationToken, + idle_timeout: Duration, + byte_budget: &Arc, +) -> Result { + let mut sent = 0_usize; + let mut last_progress = Instant::now(); + for buf in start_buffer { + let deadline = last_progress + idle_timeout; + match send_client_chunk(client_tx, buf.clone(), cancellation_token, deadline, byte_budget).await { + SendOutcome::Sent => { + sent = sent.saturating_add(1); + last_progress = Instant::now(); + } + outcome => return Err(outcome), + } + } + Ok(sent) +} + +async fn send_client_chunk( + client_tx: &Sender, + data: Bytes, + cancellation_token: &CancellationToken, + progress_deadline: Instant, + byte_budget: &Arc, +) -> SendOutcome { + if cancellation_token.is_cancelled() { + return SendOutcome::Cancelled; + } + + let permit = match reserve_byte_permit(byte_budget, data.len(), cancellation_token, progress_deadline).await { + Ok(permit) => permit, + Err(outcome) => return outcome, + }; + + let chunk = BudgetedChunk { bytes: data, _permit: permit }; + match client_tx.try_send(chunk) { + Ok(()) => SendOutcome::Sent, + Err(mpsc::error::TrySendError::Closed(_)) => SendOutcome::Closed, + Err(mpsc::error::TrySendError::Full(chunk)) => tokio::select! { + biased; + () = cancellation_token.cancelled() => SendOutcome::Cancelled, + result = tokio::time::timeout_at(progress_deadline, client_tx.send(chunk)) => match result { + Ok(Ok(())) => SendOutcome::Sent, + Ok(Err(_)) => SendOutcome::Closed, + Err(_) => SendOutcome::TimedOut, + }, + }, } } @@ -218,12 +417,16 @@ pub struct SharedStreamState { low_priority_preempted: Option, preempted_token: CancellationToken, subscribers: RwLock>, - next_subscriber_id: AtomicU64, stop_token: CancellationToken, burst_buffer: Arc>, live_notification: Arc, - task_handles: RwLock>>, + task_handles: std::sync::Mutex>>, subscriber_idle_timeout_secs: u64, + subscriber_max_duration: Option, + /// Number of joins that committed their registry entry but have not yet registered + /// their subscriber in `subscribers`. Teardown must not remove an origin while this is + /// non-zero, otherwise a join in flight would land on a stopped state. + pending_joins: AtomicUsize, } impl SharedStreamState { @@ -244,12 +447,13 @@ impl SharedStreamState { low_priority_preempted, preempted_token: CancellationToken::new(), subscribers: RwLock::new(HashMap::new()), - next_subscriber_id: AtomicU64::new(1), stop_token: CancellationToken::new(), burst_buffer: Arc::new(Mutex::new(BurstBuffer::new(burst_buffer_size_in_bytes))), live_notification: Arc::new(Notify::new()), - task_handles: RwLock::new(Vec::new()), + task_handles: std::sync::Mutex::new(Vec::new()), subscriber_idle_timeout_secs: DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, + subscriber_max_duration: None, + pending_joins: AtomicUsize::new(0), } } @@ -260,82 +464,49 @@ impl SharedStreamState { self } - async fn register_subscriber(&self, addr: &SocketAddr, cancel_token: CancellationToken) -> SharedSubscriberOwner { - let id = self.next_subscriber_id.fetch_add(1, Ordering::Relaxed); - let subscriber = SharedSubscriber { id, cancel_token }; - let previous = { - let mut subs = self.subscribers.write().await; - let previous = subs.insert(*addr, subscriber); - debug_if_enabled!( - "Shared stream subscriber added {}; total subscribers={}", - sanitize_sensitive_info(&addr.to_string()), - subs.len() - ); - previous - }; + fn increment_pending_joins(&self) { self.pending_joins.fetch_add(1, Ordering::AcqRel); } - if let Some(previous_subscriber) = previous.as_ref() { - previous_subscriber.cancel_token.cancel(); - } + fn decrement_pending_joins(&self) { self.pending_joins.fetch_sub(1, Ordering::AcqRel); } - SharedSubscriberOwner { id } + fn has_pending_joins(&self) -> bool { self.pending_joins.load(Ordering::Acquire) > 0 } + + fn lock_task_handles(&self) -> std::sync::MutexGuard<'_, Vec>> { + self.task_handles.lock().unwrap_or_else(std::sync::PoisonError::into_inner) } - async fn remove_subscriber_if_owner(&self, addr: &SocketAddr, owner: SharedSubscriberOwner) -> bool { - let mut subs = self.subscribers.write().await; - if subs.get(addr).is_none_or(|subscriber| subscriber.id != owner.id) { - return false; - } - subs.remove(addr); - true + async fn register_subscriber(&self, id: SubscriberId, addr: &SocketAddr, cancel_token: CancellationToken) { + self.subscribers.write().await.insert(id, SharedSubscriber { addr: *addr, cancel_token }); } async fn cancel_subscribers(&self) { - let subscribers = self.subscribers.read().await; - for subscriber in subscribers.values() { + for subscriber in self.subscribers.read().await.values() { subscriber.cancel_token.cancel(); } } - async fn has_no_subscribers(&self) -> bool { self.subscribers.read().await.is_empty() } - - async fn cleanup_subscriber( - state: &Arc, - manager: &SharedStreamManager, - connection_manager: &ConnectionManager, - addr: &SocketAddr, - owner: SharedSubscriberOwner, - ) { - if !state.remove_subscriber_if_owner(addr, owner).await { - return; - } - let stream_url = manager.forget_subscriber_addr(addr).await; - let is_empty = state.has_no_subscribers().await; - connection_manager.release_stream(addr).await; - connection_manager.release_provider_connection(addr).await; - if is_empty { - if let Some(stream_url) = stream_url.as_ref() { - manager.unregister(stream_url, false).await; - } - } - } - #[allow(clippy::too_many_lines)] async fn subscribe( self: &Arc, addr: &SocketAddr, - manager: Arc, + subscriber_id: SubscriberId, connection_manager: Arc, - ) -> (BoxedProviderStream, Option>) { - let (client_tx, client_rx) = mpsc::channel(self.buf_size); + mut pending_cleanup: PendingSharedSubscriberCleanup, + pending_join: PendingJoinGuard, + ) -> (BoxedProviderStream, Option>, SharedCleanupCapability) { + let (client_tx, client_rx) = mpsc::channel::(self.buf_size); let cancel_token = CancellationToken::new(); + let queue_byte_budget = self.buf_size.saturating_mul(SHARED_BURST_BYTES_PER_BUFFER_SLOT); + let byte_budget = Arc::new(Semaphore::new(queue_byte_budget)); { - let mut handles = self.task_handles.write().await; + let mut handles = self.lock_task_handles(); handles.retain(|h| !h.is_finished()); } - let owner = self.register_subscriber(addr, cancel_token.clone()).await; + self.register_subscriber(subscriber_id, addr, cancel_token.clone()).await; + pending_join.commit(); + let (start_tx, start_rx) = oneshot::channel(); + let cleanup_manager = Arc::clone(&connection_manager); let client_tx_clone = client_tx.clone(); let burst_buffer = Arc::clone(&self.burst_buffer); @@ -343,7 +514,6 @@ impl SharedStreamState { let live_notification = Arc::clone(&self.live_notification); let timeout_duration = Duration::from_secs(self.subscriber_idle_timeout_secs); let idle_check_interval = Duration::from_secs(1); - let mut last_active = Instant::now(); let mut last_lag_log = Instant::now().checked_sub(Duration::from_secs(10)).unwrap_or_else(Instant::now); let mut consecutive_lag_count: u32 = 0; let subscriber_buf_size = self.buf_size; @@ -351,20 +521,52 @@ impl SharedStreamState { let low_priority_preempted = self.low_priority_preempted.clone(); let address = *addr; let subscriber_started_at = Instant::now(); - let state = Arc::clone(self); let handle = tokio::spawn(async move { + // Wait for the response to be polled, but respect cancellation/shutdown so a + // body that is never polled cannot keep the forwarder task alive indefinitely. + tokio::select! { + () = cancel_token.cancelled() => return, + started = start_rx => { + if started.is_err() { + return; + } + } + } + // Send-progress budget begins only once the body is actually polled. + let mut last_active = Instant::now(); let (snapshot, mut next_sequence) = { let buffer = burst_buffer.lock().await; buffer.snapshot() }; - let sent_burst_chunks = send_burst_buffer(&snapshot, &client_tx_clone, &cancel_token).await; - if sent_burst_chunks > 0 { - debug_if_enabled!( - "Shared stream subscriber {} replayed {sent_burst_chunks} burst chunks after {} ms", - sanitize_sensitive_info(&address.to_string()), - subscriber_started_at.elapsed().as_millis() - ); + match send_burst_buffer(&snapshot, &client_tx_clone, &cancel_token, timeout_duration, &byte_budget).await { + Ok(sent_burst_chunks) => { + drop(snapshot); + if sent_burst_chunks > 0 { + // The replay renewed its own internal progress time; carry that + // forward so the first live send does not immediately expire. + last_active = Instant::now(); + debug_if_enabled!( + "Shared stream subscriber {} replayed {sent_burst_chunks} burst chunks after {} ms", + sanitize_sensitive_info(&address.to_string()), + subscriber_started_at.elapsed().as_millis() + ); + } + } + Err(outcome) => { + drop(snapshot); + debug!( + "Shared stream subscriber {} burst replay failed ({outcome:?}); terminating", + sanitize_sensitive_info(&address.to_string()) + ); + cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { + addr: address, + subscriber_id, + request_id: None, + owner: None, + }); + return; + } } let mut first_live_chunk_logged = false; @@ -382,7 +584,7 @@ impl SharedStreamState { let read = { let buffer = burst_buffer.lock().await; - buffer.read_from_into(next_sequence, &mut read_chunks) + buffer.read_from_into(next_sequence, &mut read_chunks, MAX_SUBSCRIBER_LIVE_BATCH_CHUNKS) }; next_sequence = read.next_sequence; if read.skipped > 0 { @@ -414,10 +616,26 @@ impl SharedStreamState { if !read_chunks.is_empty() { for data in read_chunks.drain(..) { let chunk_len = data.len(); - if !send_client_chunk(&client_tx, data, &cancel_token).await { - debug!("Shared stream client send error: {address}"); - Self::cleanup_subscriber(&state, &manager, &connection_manager, &address, owner).await; - return; + match send_client_chunk( + &client_tx, + data, + &cancel_token, + last_active + timeout_duration, + &byte_budget, + ) + .await + { + SendOutcome::Sent => {} + outcome => { + debug!("Shared stream client send error ({outcome:?}): {address}"); + cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { + addr: address, + subscriber_id, + request_id: None, + owner: None, + }); + return; + } } if !first_live_chunk_logged { debug_if_enabled!( @@ -490,12 +708,15 @@ impl SharedStreamState { sanitize_sensitive_info(&address.to_string()) ); while let Some(chunk) = fallback.next_chunk() { - if !send_client_chunk(&client_tx, chunk, &cancel_token).await { - debug!( - "Shared stream fallback send error for {}", - sanitize_sensitive_info(&address.to_string()) - ); - break; + match send_client_chunk(&client_tx, chunk, &cancel_token, last_active + timeout_duration, &byte_budget).await { + SendOutcome::Sent => last_active = Instant::now(), + outcome => { + debug!( + "Shared stream fallback send error ({outcome:?}) for {}", + sanitize_sensitive_info(&address.to_string()) + ); + break; + } } } } @@ -504,13 +725,34 @@ impl SharedStreamState { } } - Self::cleanup_subscriber(&state, &manager, &connection_manager, &address, owner).await; + cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { + addr: address, + subscriber_id, + request_id: None, + owner: None, + }); }); - self.task_handles.write().await.push(handle); + self.lock_task_handles().push(handle); let provider = self.provider_guard.as_ref().and_then(|h| h.allocation.get_provider_name()); - (convert_stream(ReceiverStream::new(client_rx).boxed()), provider) + let (permit, request_id, owner) = pending_cleanup.take_cleanup(); + ( + ReceiverStreamWrapper { + stream: ReceiverStream::new(client_rx).map(|chunk| chunk.bytes), + start: Some(start_tx), + subscriber_id, + addr: *addr, + permit, + request_id, + owner, + deadline: self.subscriber_max_duration.map(|duration| Box::pin(sleep(duration))), + released: false, + } + .boxed(), + provider, + SharedCleanupCapability::new(subscriber_id), + ) } #[allow(clippy::too_many_lines)] @@ -520,12 +762,13 @@ impl SharedStreamState { E: std::fmt::Debug + Send, { let streaming_url = stream_url.to_string(); + let origin_state = Arc::clone(self); let stop_token = self.stop_token.clone(); let burst_buffer = Arc::clone(&self.burst_buffer); let live_notification = Arc::clone(&self.live_notification); let broadcast_started_at = Instant::now(); - tokio::spawn(async move { + let broadcast_handle = tokio::spawn(async move { let mut source_stream = std::pin::pin!(bytes_stream); let mut counter = 0_usize; let idle_timeout = Duration::from_secs(STREAM_IDLE_TIMEOUT); @@ -661,21 +904,28 @@ impl SharedStreamState { sanitize_sensitive_info(&streaming_url), last_push_at.map_or(0, |t| t.elapsed().as_secs()) ); - shared_streams.unregister(&streaming_url, false).await; + shared_streams.unregister(&streaming_url, &origin_state).await; }); + + // Keep the broadcast handle so shutdown can join it instead of leaving a detached task. + // Registration is guaranteed: a short-lived synchronous lock avoids the try_write + // failure that would otherwise drop the handle under contention. + self.lock_task_handles().push(broadcast_handle); } } #[derive(Debug, Clone, Default)] struct SharedStreamsRegister { by_key: HashMap, Arc>, - key_by_addr: HashMap>, + key_by_subscriber: HashMap>, } pub struct SharedStreamManager { provider_manager: Arc, shared_streams: RwLock, - meter_uids: RwLock>, + meter_uids: std::sync::Mutex, SharedMeterEntry>>, + #[cfg(test)] + pub(crate) test_preflight_barrier: std::sync::Mutex>>, } /// The four state handles the shared-stream paths need. @@ -692,11 +942,35 @@ pub struct SharedStreamCtx<'a> { } impl SharedStreamManager { + pub async fn reserve_subscriber_cleanup( + connection_manager: &ConnectionManager, + subscriber_id: SharedSubscriberId, + addr: SocketAddr, + ) -> Result { + if connection_manager.is_shutting_down() { + warn!("Shared stream cleanup rejected during shutdown for subscriber {subscriber_id}"); + return Err(ConnectionRejectionReason::CleanupReceiverClosed); + } + match timeout(SHARED_CLEANUP_ADMISSION_TIMEOUT, connection_manager.cleanup_tx().reserve_owned()).await { + Ok(Ok(permit)) => Ok(PendingSharedSubscriberCleanup::new(permit, subscriber_id, addr)), + Ok(Err(_)) => { + warn!("Shared stream cleanup receiver closed; rejecting subscriber {subscriber_id}"); + Err(ConnectionRejectionReason::CleanupReceiverClosed) + } + Err(_) => { + warn!("Shared stream cleanup admission timed out; rejecting subscriber {subscriber_id}"); + Err(ConnectionRejectionReason::CleanupAdmissionTimeout) + } + } + } + pub fn new(provider_manager: Arc) -> Self { Self { provider_manager, shared_streams: RwLock::new(SharedStreamsRegister::default()), - meter_uids: RwLock::new(HashMap::new()), + meter_uids: std::sync::Mutex::new(HashMap::new()), + #[cfg(test)] + test_preflight_barrier: std::sync::Mutex::new(None), } } @@ -708,180 +982,241 @@ impl SharedStreamManager { self.get_shared_state(stream_url).await.map(|s| s.headers.clone()) } - pub async fn get_or_register_meter_uid(&self, stream_url: &str, uid_factory: impl FnOnce() -> u32) -> u32 { - let mut uids = self.meter_uids.write().await; - *uids.entry(stream_url.to_string()).or_insert_with(uid_factory) + pub async fn resource_counts(&self) -> (usize, usize) { + let register = self.shared_streams.read().await; + (register.by_key.len(), register.key_by_subscriber.len()) } - async fn forget_subscriber_addr(&self, addr: &SocketAddr) -> Option> { - let mut shared_streams = self.shared_streams.write().await; - shared_streams.key_by_addr.remove(addr) + fn lock_meter_uids(&self) -> std::sync::MutexGuard<'_, HashMap, SharedMeterEntry>> { + self.meter_uids.lock().unwrap_or_else(|poisoned| { + warn!("Recovering poisoned shared-stream meter registry"); + poisoned.into_inner() + }) } - async fn unregister(&self, stream_url: &str, send_stop_signal: bool) { - let shared_state_opt = { - let mut shared_streams = self.shared_streams.write().await; + pub fn reserve_meter_uid( + self: &Arc, + stream_url: &str, + uid_factory: impl FnOnce() -> u32, + ) -> (u32, Option) { + let mut uids = self.lock_meter_uids(); + if let Some(entry) = uids.get(stream_url) { + return (entry.uid, None); + } + let stream_url: Arc = Arc::from(stream_url); + let meter_uid = uid_factory(); + uids.insert(Arc::clone(&stream_url), SharedMeterEntry { uid: meter_uid, owner: None }); + drop(uids); + ( + meter_uid, + Some(PendingSharedMeterRegistration { manager: Arc::clone(self), stream_url, meter_uid, armed: true }), + ) + } - let remove_keys: Vec = shared_streams - .key_by_addr - .iter() - .filter_map(|(addr, url)| if url.as_ref() == stream_url { Some(*addr) } else { None }) - .collect(); - for k in remove_keys { - shared_streams.key_by_addr.remove(&k); - } + /// Binds the reserved meter entry to the origin that just committed its shared + /// origin. Called only while the registry write lock is held and only after the URL + /// was confirmed absent from `by_key`, so the previous origin (if any) is already + /// torn down and this new origin is the sole owner. Overwriting the owner is what + /// keeps a successor's meter from being removed by a stale predecessor teardown. + pub fn adopt_meter_uid(&self, stream_url: &str, allocation_id: AllocationId) { + let mut uids = self.lock_meter_uids(); + if let Some(entry) = uids.get_mut(stream_url) { + entry.owner = Some(allocation_id); + } + } - shared_streams.by_key.remove(stream_url) - }; - - self.meter_uids.write().await.remove(stream_url); - - if let Some(shared_state) = shared_state_opt { - let remaining = shared_state.subscribers.read().await.len(); - debug_if_enabled!( - "Unregistering shared stream {} (remaining_subscribers={remaining}, send_stop_signal={send_stop_signal})", - sanitize_sensitive_info(stream_url) - ); - - if remaining > 0 && !send_stop_signal { - shared_state.cancel_subscribers().await; - } else { - for handle in shared_state.task_handles.write().await.drain(..) { - handle.abort(); - } - } - - if let Some(provider_handle) = &shared_state.provider_guard { - self.provider_manager.release_handle(provider_handle).await; - } - - if send_stop_signal || remaining == 0 { - trace_if_enabled!("Sending shared stream stop signal {}", sanitize_sensitive_info(stream_url)); - shared_state.stop_token.cancel(); + fn remove_meter_uid_if_pending(&self, stream_url: &str, meter_uid: u32) { + let mut uids = self.lock_meter_uids(); + if let Some(entry) = uids.get(stream_url) { + if entry.uid == meter_uid && entry.owner.is_none() { + uids.remove(stream_url); } } } - pub async fn teardown_preempted_stream(&self, stream_url: &str) { - let shared_state_opt = { - let mut shared_streams = self.shared_streams.write().await; + pub fn meter_count(&self) -> usize { self.lock_meter_uids().len() } - let remove_keys: Vec = shared_streams - .key_by_addr - .iter() - .filter_map(|(addr, url)| if url.as_ref() == stream_url { Some(*addr) } else { None }) - .collect(); - for k in remove_keys { - shared_streams.key_by_addr.remove(&k); - } - - shared_streams.by_key.remove(stream_url) - }; - - self.meter_uids.write().await.remove(stream_url); - - if let Some(shared_state) = shared_state_opt { - debug_if_enabled!("Tearing down preempted shared stream {}", sanitize_sensitive_info(stream_url)); - - shared_state.preempted_token.cancel(); - shared_state.stop_token.cancel(); + fn remove_meter_uid_if_owned_by(&self, stream_url: &str, allocation_id: Option) { + let mut uids = self.lock_meter_uids(); + let remove = uids.get(stream_url).is_some_and(|entry| match allocation_id { + Some(id) => entry.owner == Some(id), + None => entry.owner.is_none(), + }); + if remove { + uids.remove(stream_url); } } - pub async fn release_connection(&self, addr: &SocketAddr, send_stop_signal: bool) { - let (stream_url, shared_state) = { - let shared_streams = self.shared_streams.read().await; - if let Some(stream_url) = shared_streams.key_by_addr.get(addr) { - (Some(stream_url.clone()), shared_streams.by_key.get(stream_url).cloned()) - } else { - (None, None) - } - }; + async fn finish_unregister(&self, stream_url: &str, state: &SharedStreamState) { + self.remove_meter_uid_if_owned_by(stream_url, state.provider_guard.as_ref().map(|handle| handle.allocation_id)); + state.cancel_subscribers().await; + state.stop_token.cancel(); + if let Some(handle) = &state.provider_guard { + self.provider_manager.release_handle(handle); + } + } - if let Some(state) = shared_state { - let (tx, is_empty, remaining) = { - let mut subs = state.subscribers.write().await; - let tx = subs.remove(addr); - let is_empty = subs.is_empty(); - (tx, is_empty, subs.len()) - }; - - let Some(client_stop_signal) = tx else { - trace_if_enabled!( - "Ignoring duplicate shared stream release for {} (already removed)", - sanitize_sensitive_info(&addr.to_string()) - ); + async fn unregister(&self, stream_url: &str, expected: &Arc) { + let mut register = self.shared_streams.write().await; + let state = { + if !register.by_key.get(stream_url).is_some_and(|state| Arc::ptr_eq(state, expected)) { return; - }; - - { - let mut shared_streams = self.shared_streams.write().await; - shared_streams.key_by_addr.remove(addr); } - - debug_if_enabled!( - "Shared stream subscriber removed {}; remaining subscribers={remaining}", - sanitize_sensitive_info(&addr.to_string()) - ); - - if is_empty { - if let Some(url) = stream_url.as_ref() { - debug_if_enabled!( - "No subscribers remain for {} after removing {}", - sanitize_sensitive_info(url), - sanitize_sensitive_info(&addr.to_string()) - ); - self.unregister(url, send_stop_signal).await; - } - } - - client_stop_signal.cancel_token.cancel(); + register.key_by_subscriber.retain(|_, url| url.as_ref() != stream_url); + register.by_key.remove(stream_url) + }; + if let Some(state) = state { + self.finish_unregister(stream_url, &state).await; } } + pub async fn teardown_preempted_stream(&self, stream_url: &str, allocation_id: u64) { + let mut register = self.shared_streams.write().await; + if !register.by_key.get(stream_url).is_some_and(|state| { + state.provider_guard.as_ref().is_some_and(|handle| handle.allocation_id == allocation_id) + }) { + return; + } + let state = { + register.key_by_subscriber.retain(|_, url| url.as_ref() != stream_url); + register.by_key.remove(stream_url) + }; + self.remove_meter_uid_if_owned_by(stream_url, Some(allocation_id)); + if let Some(state) = state { + state.preempted_token.cancel(); + state.stop_token.cancel(); + } + } + + /// Releases all subscribers on a closed transport. Playback cleanup uses the id variant. + pub async fn release_connection(&self, addr: &SocketAddr, _send_stop_signal: bool) { + let states: Vec> = { + let register = self.shared_streams.read().await; + register.by_key.values().cloned().collect() + }; + let mut ids = Vec::new(); + for state in states { + ids.extend( + state + .subscribers + .read() + .await + .iter() + .filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)), + ); + } + for id in ids { + self.release_subscriber(id).await; + } + } + + pub async fn release_subscriber(&self, subscriber_id: SubscriberId) { + let mut register = self.shared_streams.write().await; + let stopped = { + // Registry then subscribers is the shared-stream lock order. Joining and + // removing the final subscriber must not race an origin replacement. + if let Some(url) = register.key_by_subscriber.remove(&subscriber_id) { + if let Some(state) = register.by_key.get(&url).cloned() { + let mut subscribers = state.subscribers.write().await; + if let Some(subscriber) = subscribers.remove(&subscriber_id) { + subscriber.cancel_token.cancel(); + } + if subscribers.is_empty() && !state.has_pending_joins() { + register.by_key.remove(&url); + drop(subscribers); + Some((url, state)) + } else { + None + } + } else { + None + } + } else { + None + } + }; + drop(register); + self.provider_manager.release_shared_connection(subscriber_id); + if let Some((url, state)) = stopped { + self.finish_unregister(&url, &state).await; + } + } + + pub async fn shutdown(&self) { + let stopped = { + let mut register = self.shared_streams.write().await; + register.key_by_subscriber.clear(); + register.by_key.drain().collect::>() + }; + for (url, state) in stopped { + self.finish_unregister(&url, &state).await; + // Join the broadcast and forwarder tasks for this origin so teardown is + // complete before the manager (and its runtime) is dropped. Cancellation has + // already fired, so these tasks only finish their terminal cleanup. + let handles = std::mem::take(&mut *state.lock_task_handles()); + for handle in handles { + let _ = handle.await; + } + } + self.lock_meter_uids().clear(); + } + async fn subscribe_stream( &self, stream_url: &str, addr: &SocketAddr, - manager: Arc, + subscriber_id: SubscriberId, connection_manager: Arc, - ) -> Option<(BoxedProviderStream, Option>)> { - let shared_state_opt = { - let mut shared_streams = self.shared_streams.write().await; - if let Some((stream_key, shared_state)) = shared_streams - .by_key - .get_key_value(stream_url) - .map(|(stream_key, shared_state)| (Arc::clone(stream_key), Arc::clone(shared_state))) - { - shared_streams.key_by_addr.insert(*addr, stream_key); - Some(shared_state) - } else { - None - } + user_priority: i8, + connection_kind: crate::active_provider_manager::ConnectionKind, + ) -> Result>, SharedCleanupCapability)>, ConnectionRejectionReason> + { + let Some(_admission) = connection_manager.begin_admission().await else { + return Err(ConnectionRejectionReason::CleanupReceiverClosed); }; - - if let Some(shared_state) = shared_state_opt { - debug_if_enabled!( - "Responding to existing shared client stream {} {}", - sanitize_sensitive_info(&addr.to_string()), - sanitize_sensitive_info(stream_url) - ); - Some(shared_state.subscribe(addr, manager, connection_manager).await) - } else { - None + { + let register = self.shared_streams.read().await; + if !register.by_key.contains_key(stream_url) { + return Ok(None); + } } - } - - async fn register(&self, addr: &SocketAddr, stream_url: &str, shared_state: Arc) { - let mut shared_streams = self.shared_streams.write().await; - let stream_key: Arc = Arc::from(stream_url); - shared_streams.by_key.insert(Arc::clone(&stream_key), shared_state); - shared_streams.key_by_addr.insert(*addr, stream_key); - debug_if_enabled!( - "Registered shared stream {} for initial subscriber {}", - sanitize_sensitive_info(stream_url), - sanitize_sensitive_info(&addr.to_string()) - ); + #[cfg(test)] + { + let barrier = self.test_preflight_barrier.lock().ok().and_then(|g| g.clone()); + if let Some(barrier) = barrier { + barrier.wait().await; + } + } + let mut pending_cleanup = Self::reserve_subscriber_cleanup(&connection_manager, subscriber_id, *addr).await?; + // Keep the global registry lock only for the lookup and the key_by_subscriber + // commit. The origin's own subscriber/task locks are taken during subscribe, so a + // slow origin must not block joins on other URLs. + let (state, pending_join) = { + let mut register = self.shared_streams.write().await; + let Some((key, state)) = + register.by_key.get_key_value(stream_url).map(|(key, state)| (Arc::clone(key), Arc::clone(state))) + else { + pending_cleanup.disarm(); + return Ok(None); + }; + if let Err(err) = self.provider_manager.add_shared_connection( + addr, + subscriber_id, + stream_url, + user_priority, + connection_kind, + ) { + warn!("Failed joining shared stream: {}", sanitize_sensitive_info(&err)); + pending_cleanup.disarm(); + return Ok(None); + } + register.key_by_subscriber.insert(subscriber_id, key); + let pending_join = PendingJoinGuard::new(&state); + (state, pending_join) + }; + Ok(Some( + state.subscribe(addr, subscriber_id, Arc::clone(&connection_manager), pending_cleanup, pending_join).await, + )) } #[allow(clippy::too_many_arguments)] @@ -890,24 +1225,55 @@ impl SharedStreamManager { stream_url: &str, bytes_stream: S, addr: &SocketAddr, + subscriber_id: SubscriberId, headers: Vec<(String, String)>, buffer_size: usize, - provider_handle: Option, + mut provider_handle: Option, + pending_cleanup: PendingSharedSubscriberCleanup, user_priority: i8, connection_kind: crate::active_provider_manager::ConnectionKind, - ) -> Option<(BoxedProviderStream, Option>)> + ) -> Option<(BoxedProviderStream, Option>, SharedCleanupCapability)> where S: Stream> + Unpin + 'static + Send, E: std::fmt::Debug + Send, { + let _admission = ctx.connection_manager.begin_admission().await?; let registration_started_at = Instant::now(); let buf_size = CHANNEL_SIZE.max(buffer_size); let config = ctx.app_config.config.load(); let min_buffer_bytes = resolve_min_burst_buffer_bytes(&config); let low_priority_preempted = ctx.app_config.custom_stream_response.load().as_ref().and_then(|c| c.low_priority_preempted.clone()); - let shared_state = Arc::new( - SharedStreamState::new(headers, buf_size, provider_handle, min_buffer_bytes, low_priority_preempted) + let mut register = ctx.shared_stream_manager.shared_streams.write().await; + if let Some((key, existing)) = + register.by_key.get_key_value(stream_url).map(|(key, state)| (Arc::clone(key), Arc::clone(state))) + { + drop(provider_handle.take()); + if ctx + .active_provider + .add_shared_connection(addr, subscriber_id, stream_url, user_priority, connection_kind) + .is_err() + { + return None; + } + register.key_by_subscriber.insert(subscriber_id, key); + let pending_join = PendingJoinGuard::new(&existing); + drop(register); + let response = existing + .subscribe(addr, subscriber_id, Arc::clone(ctx.connection_manager), pending_cleanup, pending_join) + .await; + return Some(response); + } + let handle = provider_handle.as_ref().and_then(|managed| managed.handle())?; + if !ctx.active_provider.make_shared_connection(handle, stream_url, subscriber_id) { + drop(register); + drop(provider_handle.take()); + return None; + } + let allocation_id = handle.allocation_id; + let raw_handle = provider_handle.and_then(|mut managed| managed.disarm())?; + let mut shared_state = + SharedStreamState::new(headers, buf_size, Some(raw_handle), min_buffer_bytes, low_priority_preempted) .with_subscriber_idle_timeout_secs( config .reverse_proxy @@ -916,72 +1282,59 @@ impl SharedStreamManager { .map_or(DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, |stream| { stream.shared_subscriber_idle_timeout_secs }), - ), - ); - ctx.shared_stream_manager.register(addr, stream_url, Arc::clone(&shared_state)).await; - ctx.active_provider.make_shared_connection(addr, stream_url).await; - let subscribed_stream = - Self::subscribe_shared_stream(ctx, stream_url, addr, user_priority, connection_kind).await; + ); + shared_state.subscriber_max_duration = + config.sleep_timer_mins.filter(|mins| *mins > 0).map(|mins| Duration::from_secs(u64::from(mins) * 60)); + let shared_state = Arc::new(shared_state); + let stream_key: Arc = Arc::from(stream_url); + register.by_key.insert(Arc::clone(&stream_key), Arc::clone(&shared_state)); + register.key_by_subscriber.insert(subscriber_id, stream_key); + ctx.shared_stream_manager.adopt_meter_uid(stream_url, allocation_id); + let pending_join = PendingJoinGuard::new(&shared_state); + drop(register); + let subscribed_stream = shared_state + .subscribe(addr, subscriber_id, Arc::clone(ctx.connection_manager), pending_cleanup, pending_join) + .await; debug_if_enabled!( "Shared stream startup register+subscribe completed for {} in {} ms", sanitize_sensitive_info(stream_url), registration_started_at.elapsed().as_millis() ); - if subscribed_stream.is_some() { - shared_state.broadcast(stream_url, bytes_stream, Arc::clone(ctx.shared_stream_manager)); - debug_if_enabled!( - "Created shared provider stream {} (channel_capacity={buf_size}, burst_buffer_min={min_buffer_bytes} bytes)", - sanitize_sensitive_info(stream_url) - ); - } else { - warn!( - "Shared stream subscribe failed for {}; broadcaster will not start", - sanitize_sensitive_info(stream_url) - ); - } - subscribed_stream + shared_state.broadcast(stream_url, bytes_stream, Arc::clone(ctx.shared_stream_manager)); + Some(subscribed_stream) } pub async fn subscribe_shared_stream( ctx: SharedStreamCtx<'_>, stream_url: &str, addr: &SocketAddr, + subscriber_id: SubscriberId, user_priority: i8, connection_kind: crate::active_provider_manager::ConnectionKind, - ) -> Option<(BoxedProviderStream, Option>)> { - let manager = Arc::clone(ctx.shared_stream_manager); - let connection_manager = Arc::clone(ctx.connection_manager); - if let Some(result) = - ctx.shared_stream_manager.subscribe_stream(stream_url, addr, manager, connection_manager).await - { - match ctx.active_provider.add_shared_connection(addr, stream_url, user_priority, connection_kind).await { - Ok(()) => Some(result), - Err(err) => { - warn!( - "Rolling back shared stream subscriber {} for {}: {}", - sanitize_sensitive_info(&addr.to_string()), - sanitize_sensitive_info(stream_url), - sanitize_sensitive_info(&err) - ); - ctx.shared_stream_manager.release_connection(addr, true).await; - None - } - } - } else { - None - } + ) -> Result>, SharedCleanupCapability)>, ConnectionRejectionReason> + { + ctx.shared_stream_manager + .subscribe_stream( + stream_url, + addr, + subscriber_id, + Arc::clone(ctx.connection_manager), + user_priority, + connection_kind, + ) + .await } } #[cfg(test)] mod tests { use super::{ - send_client_chunk, BurstBuffer, SharedStreamManager, SharedStreamState, CHANNEL_SIZE, - MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES, + send_client_chunk, BudgetedChunk, BurstBuffer, SendOutcome, SharedStreamManager, SharedStreamState, + CHANNEL_SIZE, MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES, SHARED_CLEANUP_ADMISSION_TIMEOUT, }; use crate::{ ActiveProviderManager, ActiveUserConnectionParams, ActiveUserManager, ConnectionKind, ConnectionManager, - EventManager, + ConnectionRejectionReason, EventManager, ManagedProviderHandle, }; use arc_swap::{ArcSwap, ArcSwapOption}; use bytes::Bytes; @@ -992,17 +1345,21 @@ mod tests { }; use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; use tokio::{ - sync::mpsc, - time::{timeout, Duration}, + sync::{mpsc, Semaphore}, + time::{timeout, Duration, Instant}, }; use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::CancellationToken; use tuliprox_core::{ - model::{AppConfig, Config, ConfigInput, Fingerprint, MediaToolCapabilities, SourcesConfig}, + model::{ + AppConfig, Config, ConfigInput, Fingerprint, MediaToolCapabilities, SharedSubscriberId, SourcesConfig, + }, utils::FileLockManager, }; - fn create_test_app_config() -> AppConfig { + fn create_test_app_config() -> AppConfig { create_test_app_config_with_conns(1) } + + fn create_test_app_config_with_conns(max_connections: u16) -> AppConfig { let input = Arc::new(ConfigInput { id: 1, name: "provider_1".intern(), @@ -1013,7 +1370,7 @@ mod tests { password: Some("pass1".to_string()), enabled: true, priority: 0, - max_connections: 1, + max_connections, method: InputFetchMethod::default(), aliases: None, ..ConfigInput::default() @@ -1081,219 +1438,374 @@ mod tests { (provider_manager, user_manager, shared_manager, connection_manager) } - #[allow(clippy::too_many_arguments)] - async fn register_active_shared_test_stream( - provider_manager: &Arc, - user_manager: &Arc, - shared_manager: &Arc, - connection_manager: &Arc, - state: Arc, - stream_url: &str, + async fn register_user( + users: &ActiveUserManager, addr: SocketAddr, uid: u32, - ) { - let input_name = "provider_1".intern(); + username: &str, + stream_url: &str, + ) -> Result<(), Box> { + let fingerprint = Fingerprint::new(username.to_string(), username.to_string(), addr); let channel = create_test_stream_channel(stream_url); - let fingerprint = Fingerprint::new("client-key".to_string(), "127.0.0.1".to_string(), addr); - - provider_manager - .acquire_connection(&input_name, &addr, 0, ConnectionKind::Normal) - .await - .unwrap_or_else(|| panic!("provider allocation expected")); - provider_manager.make_shared_connection(&addr, stream_url).await; - shared_manager.register(&addr, stream_url, state).await; - - connection_manager.add_connection(&addr).await; - let stream_info = user_manager + let stream = users .update_connection(ActiveUserConnectionParams { uid, meter_uid: 0, - username: "user1", - max_connections: 1, + username, + max_connections: 5, soft_connections: 0, connection_kind: ConnectionKind::Normal, priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: input_name, + provider: "provider_1".intern(), stream_channel: &channel, user_agent: Cow::Borrowed("test"), session_token: None, }) .await - .unwrap_or_else(|| panic!("active user stream expected")); - assert_eq!(stream_info.addr, addr); + .ok_or("user stream missing")?; + assert_eq!(stream.uid, uid); + Ok(()) } #[tokio::test] - async fn replacing_subscriber_token_cancels_previous_subscriber() { - let state = SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None); - let addr: SocketAddr = "127.0.0.1:41003".parse().unwrap_or_else(|_| unreachable!()); - let old_token = CancellationToken::new(); - let new_token = CancellationToken::new(); - - let old_owner = state.register_subscriber(&addr, old_token.clone()).await; - assert_eq!(old_owner.id, 1); - - let new_owner = state.register_subscriber(&addr, new_token).await; - assert_eq!(new_owner.id, 2); - assert!(old_token.is_cancelled(), "replaced subscriber token must be cancelled"); + async fn subscribers_on_same_socket_do_not_replace_each_other() -> Result<(), Box> { + let state = SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None); + let addr = "127.0.0.1:41003".parse()?; + let first = CancellationToken::new(); + let second = CancellationToken::new(); + state.register_subscriber(SharedSubscriberId::from_stream_uid(1), &addr, first.clone()).await; + state.register_subscriber(SharedSubscriberId::from_stream_uid(2), &addr, second.clone()).await; + assert_eq!(state.subscribers.read().await.len(), 2); + assert!(!first.is_cancelled()); + assert!(!second.is_cancelled()); + Ok(()) } #[tokio::test] - async fn send_client_chunk_returns_when_cancelled_while_queue_is_full() { - let (tx, _rx) = mpsc::channel(1); - tx.send(Bytes::from_static(b"queued")).await.unwrap_or_else(|_| panic!("initial send should fill queue")); - let cancel_token = CancellationToken::new(); - cancel_token.cancel(); - - let result = - timeout(Duration::from_secs(1), send_client_chunk(&tx, Bytes::from_static(b"blocked"), &cancel_token)) - .await; - - assert!(result.is_ok(), "cancelled send must not wait for channel capacity"); - assert!(!result.unwrap_or_else(|_| unreachable!())); - } - - #[tokio::test] - async fn cleanup_subscriber_releases_user_stream_and_provider_subscriber() { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let (provider_manager, user_manager, shared_manager, connection_manager) = - create_test_connection_manager(&app_cfg, &event_manager); - - let stream_url = "https://example.invalid/live/shared.ts"; - let addr: SocketAddr = "127.0.0.1:41004".parse().unwrap_or_else(|_| unreachable!()); - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); - let owner = state.register_subscriber(&addr, CancellationToken::new()).await; - register_active_shared_test_stream( - &provider_manager, - &user_manager, - &shared_manager, - &connection_manager, - Arc::clone(&state), - stream_url, - addr, - 1, + async fn send_client_chunk_returns_when_cancelled_while_queue_is_full() -> Result<(), Box> { + let (tx, _rx) = mpsc::channel::(1); + let byte_budget = Arc::new(Semaphore::new(1024)); + let queued_permit = byte_budget.clone().acquire_many_owned(6).await?; + tx.send(BudgetedChunk { bytes: Bytes::from_static(b"queued"), _permit: queued_permit }).await?; + let cancel = CancellationToken::new(); + cancel.cancel(); + let outcome = timeout( + Duration::from_secs(1), + send_client_chunk( + &tx, + Bytes::from_static(b"blocked"), + &cancel, + Instant::now() + Duration::from_secs(1), + &byte_budget, + ), ) - .await; - assert_eq!(user_manager.active_streams().await.len(), 1); - assert_eq!(provider_manager.get_provider_connections_count().await, 1); - - SharedStreamState::cleanup_subscriber(&state, &shared_manager, &connection_manager, &addr, owner).await; - - assert!(user_manager.active_streams().await.is_empty()); - assert_eq!(provider_manager.get_provider_connections_count().await, 0); - let register = shared_manager.shared_streams.read().await; - assert!(!register.by_key.contains_key(stream_url)); - assert!(!register.key_by_addr.contains_key(&addr)); + .await?; + assert_eq!(outcome, SendOutcome::Cancelled); + Ok(()) } #[tokio::test] - async fn stale_replaced_subscriber_cannot_cleanup_current_subscriber() { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let (provider_manager, user_manager, shared_manager, connection_manager) = - create_test_connection_manager(&app_cfg, &event_manager); - - let stream_url = "https://example.invalid/live/replaced.ts"; - let addr: SocketAddr = "127.0.0.1:41005".parse().unwrap_or_else(|_| unreachable!()); - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); - let stale_owner = state.register_subscriber(&addr, CancellationToken::new()).await; - let current_owner = state.register_subscriber(&addr, CancellationToken::new()).await; - assert_ne!(stale_owner, current_owner); - - register_active_shared_test_stream( - &provider_manager, - &user_manager, - &shared_manager, - &connection_manager, - Arc::clone(&state), - stream_url, - addr, - 2, + async fn send_client_chunk_respects_progress_deadline_while_queue_is_full() -> Result<(), Box> + { + let (tx, _rx) = mpsc::channel::(1); + let byte_budget = Arc::new(Semaphore::new(1024)); + let queued_permit = byte_budget.clone().acquire_many_owned(6).await?; + tx.send(BudgetedChunk { bytes: Bytes::from_static(b"queued"), _permit: queued_permit }).await?; + let cancel = CancellationToken::new(); + let deadline = Instant::now() + Duration::from_millis(50); + let outcome = timeout( + Duration::from_secs(1), + send_client_chunk(&tx, Bytes::from_static(b"blocked"), &cancel, deadline, &byte_budget), ) - .await; - - SharedStreamState::cleanup_subscriber(&state, &shared_manager, &connection_manager, &addr, stale_owner).await; - - assert_eq!(user_manager.active_streams().await.len(), 1); - assert_eq!(provider_manager.get_provider_connections_count().await, 1); - let register = shared_manager.shared_streams.read().await; - assert!(register.by_key.contains_key(stream_url)); - assert!(register.key_by_addr.contains_key(&addr)); + .await?; + assert_eq!(outcome, SendOutcome::TimedOut); + Ok(()) } #[tokio::test] - async fn unregister_source_end_cancels_remaining_subscriber_tasks_without_aborting() { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(provider_manager)); + async fn send_client_chunk_respects_byte_budget() -> Result<(), Box> { + let (tx, _rx) = mpsc::channel::(16); + // Byte budget of 4 bytes cannot accommodate a 6-byte chunk, so the acquire must + // time out rather than exceed the budget. + let byte_budget = Arc::new(Semaphore::new(4)); + let cancel = CancellationToken::new(); + let deadline = Instant::now() + Duration::from_millis(50); + let outcome = timeout( + Duration::from_secs(1), + send_client_chunk(&tx, Bytes::from_static(b"123456"), &cancel, deadline, &byte_budget), + ) + .await?; + assert_eq!(outcome, SendOutcome::TimedOut); + Ok(()) + } - let stream_url = "https://example.invalid/live/source-ended.ts"; - let addr: SocketAddr = "127.0.0.1:41006".parse().unwrap_or_else(|_| unreachable!()); - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); - let cancel_token = CancellationToken::new(); - state.register_subscriber(&addr, cancel_token.clone()).await; - - let (tx, rx) = tokio::sync::oneshot::channel(); - { - let mut handles = state.task_handles.write().await; - handles.push(tokio::spawn(async move { - cancel_token.cancelled().await; - let _ = tx.send(()); - })); - } - shared_manager.register(&addr, stream_url, Arc::clone(&state)).await; - - shared_manager.unregister(stream_url, false).await; - - assert!( - timeout(Duration::from_secs(1), rx).await.is_ok(), - "source-end unregister must cancel subscriber tasks instead of aborting them" + #[tokio::test] + async fn shared_clients_on_same_socket_keep_independent_streams_and_capacity( + ) -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41004".parse()?; + let first_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let second_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/shared.ts"; + register_user(&users, addr, first_id.stream_uid(), "first", url).await?; + let allocation = ManagedProviderHandle::new( + Arc::clone(&providers), + providers + .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) + .ok_or("provider allocation missing")?, ); + let ctx = super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + let (tx, rx) = mpsc::channel::>(8); + let pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, first_id, addr) + .await + .map_err(|_| "shared cleanup admission failed")?; + let (mut first, _, _) = SharedStreamManager::register_shared_stream( + ctx, + url, + ReceiverStream::new(rx), + &addr, + first_id, + Vec::new(), + 8, + Some(allocation), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("first subscription missing")?; + let (mut second, _, _) = + SharedStreamManager::subscribe_shared_stream(ctx, url, &addr, second_id, 0, ConnectionKind::Normal) + .await + .map_err(|_| "second shared cleanup admission failed")? + .ok_or("second subscription missing")?; + register_user(&users, addr, second_id.stream_uid(), "second", url).await?; + + tx.send(Ok(Bytes::from_static(b"first chunk"))).await?; + assert_eq!( + timeout(Duration::from_secs(2), first.next()).await?.ok_or("first ended")??, + Bytes::from_static(b"first chunk") + ); + assert_eq!( + timeout(Duration::from_secs(2), second.next()).await?.ok_or("second ended")??, + Bytes::from_static(b"first chunk") + ); + assert_eq!(users.active_streams().await.len(), 2); + assert_eq!(providers.get_provider_connections_count(), 1); + + drop(first); + timeout(Duration::from_secs(2), async { + while users.active_streams().await.len() != 1 { + tokio::task::yield_now().await; + } + }) + .await?; + // A delayed duplicate cleanup cannot remove the surviving subscriber. + connections.send_cleanup(crate::CleanupEvent::ReleaseSharedSubscriber { + addr, + subscriber_id: first_id, + request_id: None, + owner: None, + }); + tx.send(Ok(Bytes::from_static(b"second chunk"))).await?; + assert_eq!( + timeout(Duration::from_secs(2), second.next()).await?.ok_or("survivor ended")??, + Bytes::from_static(b"second chunk") + ); + assert_eq!(providers.get_provider_connections_count(), 1); + assert_eq!(users.active_streams().await.first().map(|stream| stream.uid), Some(second_id.stream_uid())); + + drop(second); + timeout(Duration::from_secs(2), async { + while !users.active_streams().await.is_empty() || providers.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await?; + assert!(manager.get_shared_state(url).await.is_none()); + Ok(()) } #[tokio::test] - async fn test_duplicate_release_connection_is_idempotent_with_remaining_subscribers() { + async fn dropping_unpolled_shared_response_releases_its_subscription() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41005".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/unpolled.ts"; + register_user(&users, addr, id.stream_uid(), "first", url).await?; + let allocation = ManagedProviderHandle::new( + Arc::clone(&providers), + providers + .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) + .ok_or("provider allocation missing")?, + ); + let pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr) + .await + .map_err(|_| "shared cleanup admission failed")?; + let response = SharedStreamManager::register_shared_stream( + super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(allocation), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("subscription missing")?; + drop(response); + timeout(Duration::from_secs(2), async { + while !users.active_streams().await.is_empty() || providers.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await?; + assert!(manager.shared_streams.read().await.key_by_subscriber.is_empty()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn saturated_cleanup_queue_rejects_shared_subscriber_before_registration( + ) -> Result<(), Box> { let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(provider_manager)); - - let stream_url = "https://example.invalid/live/stream.ts"; - let addr_1: SocketAddr = "127.0.0.1:41001".parse().unwrap_or_else(|_| unreachable!()); - let addr_2: SocketAddr = "127.0.0.1:41002".parse().unwrap_or_else(|_| unreachable!()); - - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); - - { - let mut reg = shared_manager.shared_streams.write().await; - reg.by_key.insert(Arc::from(stream_url), Arc::clone(&state)); - reg.key_by_addr.insert(addr_1, Arc::from(stream_url)); - reg.key_by_addr.insert(addr_2, Arc::from(stream_url)); + let events = Arc::new(EventManager::new()); + let (_, _, _, connections) = create_test_connection_manager(&app_cfg, &events); + let cleanup_tx = connections.cleanup_tx(); + let available = cleanup_tx.capacity(); + let mut permits = Vec::with_capacity(available); + for _ in 0..available { + permits.push(cleanup_tx.clone().try_reserve_owned()?); } - state.register_subscriber(&addr_1, CancellationToken::new()).await; - state.register_subscriber(&addr_2, CancellationToken::new()).await; + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let addr = "127.0.0.1:41008".parse()?; + let subscriber_id = SharedSubscriberId::from_stream_uid(88); + let subscribe_connections = Arc::clone(&connections); + let subscribe = tokio::spawn(async move { + SharedStreamManager::reserve_subscriber_cleanup(&subscribe_connections, subscriber_id, addr).await + }); - shared_manager.release_connection(&addr_1, false).await; + tokio::task::yield_now().await; + tokio::time::advance(SHARED_CLEANUP_ADMISSION_TIMEOUT + Duration::from_millis(1)).await; + assert!(subscribe.await?.is_err(), "saturated cleanup admission must reject the subscriber"); + assert!(state.subscribers.read().await.is_empty(), "rejected admission must not register a subscriber"); + assert!(state.lock_task_handles().is_empty(), "rejected admission must not spawn a forwarding task"); + + drop(permits); + Ok(()) + } + + #[tokio::test] + async fn shared_timeout_ends_only_its_subscriber() -> Result<(), Box> { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41007".parse()?; + let first = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let second = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url: Arc = Arc::from("https://example.invalid/timeout.ts"); + register_user(&users, addr, first.stream_uid(), "same-user", &url).await?; + register_user(&users, addr, second.stream_uid(), "same-user", &url).await?; + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let surviving_token = CancellationToken::new(); + state.register_subscriber(first, &addr, CancellationToken::new()).await; + state.register_subscriber(second, &addr, surviving_token.clone()).await; { - let subs = state.subscribers.read().await; - assert_eq!(subs.len(), 1); - assert!(subs.contains_key(&addr_2)); + let mut register = manager.shared_streams.write().await; + register.by_key.insert(Arc::clone(&url), state); + register.key_by_subscriber.insert(first, Arc::clone(&url)); + register.key_by_subscriber.insert(second, Arc::clone(&url)); } + let permit = connections.cleanup_tx().reserve_owned().await.ok(); + let mut response = super::ReceiverStreamWrapper { + stream: futures::stream::pending::(), + start: None, + subscriber_id: first, + addr, + permit, + request_id: None, + owner: None, + deadline: Some(Box::pin(tokio::time::sleep(Duration::ZERO))), + released: false, + }; + assert!(timeout(Duration::from_secs(2), response.next()).await?.is_none()); + timeout(Duration::from_secs(2), async { + while users.active_streams().await.len() != 1 { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(users.active_streams().await.first().map(|stream| stream.uid), Some(second.stream_uid())); + assert!(!surviving_token.is_cancelled()); + assert!(manager.get_shared_state(&url).await.is_some()); + Ok(()) + } - shared_manager.release_connection(&addr_1, false).await; + #[tokio::test] + async fn old_origin_cannot_unregister_replacement() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, _, manager, _) = create_test_connection_manager(&app_cfg, &events); + let url = "https://example.invalid/live/replaced.ts"; + let old = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let current = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + manager.shared_streams.write().await.by_key.insert(Arc::from(url), Arc::clone(¤t)); + manager.unregister(url, &old).await; + assert!(manager.get_shared_state(url).await.is_some_and(|state| Arc::ptr_eq(&state, ¤t))); + assert!(!current.stop_token.is_cancelled()); + } + + #[tokio::test] + async fn duplicate_subscriber_release_preserves_other_channel_on_same_socket( + ) -> Result<(), Box> { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, _, manager, _) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41006".parse()?; + let first = SharedSubscriberId::from_stream_uid(1); + let second = SharedSubscriberId::from_stream_uid(2); + let first_url: Arc = Arc::from("https://example.invalid/first.ts"); + let second_url: Arc = Arc::from("https://example.invalid/second.ts"); + let state_a = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let state_b = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let surviving_token = CancellationToken::new(); + state_a.register_subscriber(first, &addr, CancellationToken::new()).await; + state_b.register_subscriber(second, &addr, surviving_token.clone()).await; { - let subs = state.subscribers.read().await; - assert_eq!(subs.len(), 1); - assert!(subs.contains_key(&addr_2)); + let mut register = manager.shared_streams.write().await; + register.by_key.insert(Arc::clone(&first_url), state_a); + register.by_key.insert(Arc::clone(&second_url), state_b); + register.key_by_subscriber.insert(first, Arc::clone(&first_url)); + register.key_by_subscriber.insert(second, Arc::clone(&second_url)); } - - let reg = shared_manager.shared_streams.read().await; - assert!(reg.by_key.contains_key(stream_url)); + manager.release_subscriber(first).await; + manager.release_subscriber(first).await; + assert!(manager.get_shared_state(&first_url).await.is_none()); + assert!(manager.get_shared_state(&second_url).await.is_some()); + assert!(!surviving_token.is_cancelled()); + manager.release_connection(&addr, true).await; + assert!(surviving_token.is_cancelled()); + Ok(()) } #[test] @@ -1312,7 +1824,7 @@ mod tests { buffer.push(Bytes::from_static(b"abcde")); let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); assert_eq!(buffer.current_bytes, 10); assert_eq!(read.skipped, 1); @@ -1329,7 +1841,7 @@ mod tests { } let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); assert_eq!(buffer.buffer.len(), max_chunks); assert_eq!(buffer.current_bytes, max_chunks); @@ -1346,7 +1858,7 @@ mod tests { } let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); assert_eq!(buffer.buffer.len(), max_chunks); assert_eq!(buffer.current_bytes, MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES * max_chunks); @@ -1360,7 +1872,7 @@ mod tests { buffer.push(Bytes::from_static(b"12345678")); let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); assert_eq!(buffer.buffer.len(), 1); assert_eq!(buffer.current_bytes, 8); @@ -1376,7 +1888,7 @@ mod tests { buffer.push(chunk); let mut chunks = Vec::new(); - let _read = buffer.read_from_into(0, &mut chunks); + let _read = buffer.read_from_into(0, &mut chunks, usize::MAX); let Some(read_chunk) = chunks.first() else { panic!("expected one buffered chunk"); }; @@ -1384,6 +1896,28 @@ mod tests { assert_eq!(read_chunk.as_ptr(), ptr); } + #[test] + fn test_burst_buffer_live_batch_is_chunk_bounded() { + let mut buffer = BurstBuffer::new(4096); + for _ in 0..10 { + buffer.push(Bytes::from_static(b"x")); + } + + let mut chunks = Vec::new(); + let first = buffer.read_from_into(0, &mut chunks, 4); + assert_eq!(chunks.len(), 4); + assert_eq!(first.next_sequence, 4); + assert_eq!(first.skipped, 0); + + let second = buffer.read_from_into(first.next_sequence, &mut chunks, 4); + assert_eq!(chunks.len(), 4); + assert_eq!(second.next_sequence, 8); + + let third = buffer.read_from_into(second.next_sequence, &mut chunks, 4); + assert_eq!(chunks.len(), 2); + assert_eq!(third.next_sequence, 10); + } + #[tokio::test] async fn test_duplicate_release_connection_is_idempotent_with_single_subscriber() { let app_cfg = create_test_app_config(); @@ -1393,21 +1927,22 @@ mod tests { let stream_url = "https://example.invalid/live/single.ts"; let addr_1: SocketAddr = "127.0.0.1:42001".parse().unwrap_or_else(|_| unreachable!()); + let id = SharedSubscriberId::from_stream_uid(1); let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); { let mut reg = shared_manager.shared_streams.write().await; reg.by_key.insert(Arc::from(stream_url), Arc::clone(&state)); - reg.key_by_addr.insert(addr_1, Arc::from(stream_url)); + reg.key_by_subscriber.insert(id, Arc::from(stream_url)); } - state.register_subscriber(&addr_1, CancellationToken::new()).await; + state.register_subscriber(id, &addr_1, CancellationToken::new()).await; shared_manager.release_connection(&addr_1, false).await; { let reg = shared_manager.shared_streams.read().await; assert!(!reg.by_key.contains_key(stream_url)); - assert!(!reg.key_by_addr.contains_key(&addr_1)); + assert!(!reg.key_by_subscriber.contains_key(&id)); } { let subs = state.subscribers.read().await; @@ -1418,7 +1953,7 @@ mod tests { { let reg = shared_manager.shared_streams.read().await; assert!(!reg.by_key.contains_key(stream_url)); - assert!(!reg.key_by_addr.contains_key(&addr_1)); + assert!(!reg.key_by_subscriber.contains_key(&id)); } { let subs = state.subscribers.read().await; @@ -1445,7 +1980,15 @@ mod tests { let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, Some(low_priority_fallback))); - let (mut stream, _provider) = state.subscribe(&addr, Arc::clone(&shared_manager), connection_manager).await; + let subscriber_id = SharedSubscriberId::from_stream_uid(1); + let Ok(pending_cleanup) = + SharedStreamManager::reserve_subscriber_cleanup(&connection_manager, subscriber_id, addr).await + else { + panic!("shared subscriber admission failed"); + }; + let (mut stream, _provider, _capability) = state + .subscribe(&addr, subscriber_id, connection_manager, pending_cleanup, super::PendingJoinGuard::new(&state)) + .await; state.preempted_token.cancel(); drop(state); @@ -1460,6 +2003,35 @@ mod tests { assert!(!chunk.is_empty(), "fallback chunk must contain MPEG-TS bytes"); } + #[tokio::test(start_paused = true)] + async fn shared_subscription_admission_has_deadline_and_no_unprotected_fallback( + ) -> Result<(), Box> { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, _, _, connections) = create_test_connection_manager(&app_cfg, &events); + let addr: SocketAddr = "127.0.0.1:43002".parse()?; + let id = SharedSubscriberId::from_stream_uid(1); + + // Saturate the cleanup queue so the admission await blocks; the shared admission + // deadline must reject the subscriber instead of hanging or falling back. + let pending = || crate::CleanupEvent::Defer(Box::pin(std::future::pending::<()>())); + connections.send_cleanup(pending()); + for _ in 0..4096 { + connections.send_cleanup(pending()); + } + + let admission = tokio::spawn({ + let connections = Arc::clone(&connections); + async move { SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await } + }); + tokio::task::yield_now().await; + tokio::task::yield_now().await; + tokio::time::advance(super::SHARED_CLEANUP_ADMISSION_TIMEOUT + Duration::from_secs(1)).await; + let result = admission.await?; + assert!(result.is_err(), "saturated shared subscription admission must be rejected, not fall back"); + Ok(()) + } + #[tokio::test] async fn broadcast_does_not_emit_shared_stream_health_events() { let app_cfg = create_test_app_config(); @@ -1505,4 +2077,557 @@ mod tests { "broadcast shutdown must not emit runtime events" ); } + + #[tokio::test] + async fn shared_origin_abort_before_user_registration_releases_physical_and_logical_request( + ) -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, _manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41010".parse()?; + let owner = "shared-owner-1"; + let handle = providers + .acquire_connection_with_lease_for_session( + &"provider_1".intern(), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(crate::active_provider_manager::PlaybackLeaseRef::new( + owner, + tuliprox_core::model::PlaybackKind::LiveTs, + )), + ) + .ok_or("provider allocation missing")?; + let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; + let managed = ManagedProviderHandle::new(Arc::clone(&providers), handle); + let subscriber_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, subscriber_id, addr) + .await + .map_err(|err| format!("shared cleanup admission failed: {err}"))?; + pending_cleanup.set_provider_request_identity(owner, request_id); + assert_eq!(providers.get_provider_connections_count(), 1); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).starting, 1); + + drop(managed); + drop(pending_cleanup); + + timeout(Duration::from_secs(2), async { + while providers.provider_lease_usage(&"provider_1".intern()).total() != 0 { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); + assert!(users.active_streams().await.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn shared_origin_registration_rejected_releases_preacquired_handle() -> Result<(), Box> + { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41011".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/rejected_preacquired.ts"; + let owner = "rejected-preacquired-owner"; + let handle = providers + .acquire_connection_with_lease_for_session( + &"provider_1".intern(), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(crate::active_provider_manager::PlaybackLeaseRef::new( + owner, + tuliprox_core::model::PlaybackKind::LiveTs, + )), + ) + .ok_or("provider allocation missing")?; + let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; + let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); + assert_eq!(providers.get_provider_connections_count(), 1); + + let dummy_state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + manager.shared_streams.write().await.by_key.insert(Arc::from(url), dummy_state); + + let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + pending_cleanup.set_provider_request_identity(owner, request_id); + let res = SharedStreamManager::register_shared_stream( + super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(managed_handle), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await; + + assert!(res.is_none()); + timeout(Duration::from_secs(2), async { + while providers.get_provider_connections_count() != 0 + || providers.provider_lease_usage(&"provider_1".intern()).total() != 0 + { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); + Ok(()) + } + + #[tokio::test] + async fn shared_origin_abort_waiting_for_registry_lock_releases_handle() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41012".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/abort_waiting.ts"; + let owner = "abort-waiting-owner"; + + let lock_guard = manager.shared_streams.write().await; + + let handle = providers + .acquire_connection_with_lease_for_session( + &"provider_1".intern(), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(crate::active_provider_manager::PlaybackLeaseRef::new( + owner, + tuliprox_core::model::PlaybackKind::LiveTs, + )), + ) + .ok_or("provider allocation missing")?; + let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; + let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); + assert_eq!(providers.get_provider_connections_count(), 1); + + let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + pending_cleanup.set_provider_request_identity(owner, request_id); + + let register_task = tokio::spawn({ + let app_cfg = Arc::clone(&app_cfg); + let manager = Arc::clone(&manager); + let providers = Arc::clone(&providers); + let connections = Arc::clone(&connections); + async move { + let ctx = super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + SharedStreamManager::register_shared_stream( + ctx, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(managed_handle), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await + } + }); + + tokio::task::yield_now().await; + + register_task.abort(); + let _ = register_task.await; + + drop(lock_guard); + + timeout(Duration::from_secs(2), async { + while providers.get_provider_connections_count() != 0 + || providers.provider_lease_usage(&"provider_1".intern()).total() != 0 + { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); + Ok(()) + } + + #[tokio::test] + async fn concurrent_shared_origin_creation_releases_only_losing_allocation( + ) -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config_with_conns(2)); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr_1 = "127.0.0.1:41013".parse()?; + let addr_2 = "127.0.0.1:41014".parse()?; + let id_1 = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let id_2 = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/concurrent_origin.ts"; + + let handle_1 = providers + .acquire_connection(&"provider_1".intern(), &addr_1, 0, ConnectionKind::Normal) + .ok_or("first provider allocation missing")?; + let managed_1 = ManagedProviderHandle::new(Arc::clone(&providers), handle_1); + let pending_1 = SharedStreamManager::reserve_subscriber_cleanup(&connections, id_1, addr_1).await?; + let (first_stream, _, _) = SharedStreamManager::register_shared_stream( + super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr_1, + id_1, + Vec::new(), + 8, + Some(managed_1), + pending_1, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("first origin registration failed")?; + + assert_eq!(providers.get_provider_connections_count(), 1); + + let handle_2 = providers + .acquire_connection(&"provider_1".intern(), &addr_2, 0, ConnectionKind::Normal) + .ok_or("second provider allocation missing")?; + let managed_2 = ManagedProviderHandle::new(Arc::clone(&providers), handle_2); + assert_eq!(providers.get_provider_connections_count(), 2); + + let pending_2 = SharedStreamManager::reserve_subscriber_cleanup(&connections, id_2, addr_2).await?; + let (second_stream, _, _) = SharedStreamManager::register_shared_stream( + super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr_2, + id_2, + Vec::new(), + 8, + Some(managed_2), + pending_2, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("second subscriber join failed")?; + + assert_eq!(providers.get_provider_connections_count(), 1); + + drop(first_stream); + drop(second_stream); + timeout(Duration::from_secs(2), async { + while providers.get_provider_connections_count() != 0 || !users.active_streams().await.is_empty() { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + Ok(()) + } + + #[tokio::test] + async fn shutdown_rejects_waiting_admission_and_drains_owned_cleanup() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (_providers, _users, _manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41015".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let initial_capacity = connections.cleanup_tx().capacity(); + let held_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + let admission = connections.begin_admission().await.ok_or("admission unexpectedly closed")?; + + let shutdown = tokio::spawn({ + let connections = Arc::clone(&connections); + async move { connections.shutdown().await } + }); + timeout(Duration::from_secs(2), async { + while !connections.is_shutting_down() { + tokio::task::yield_now().await; + } + }) + .await?; + assert!(connections.is_shutting_down()); + + let res = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await; + assert!( + matches!(res, Err(ConnectionRejectionReason::CleanupReceiverClosed)), + "waiting admission must be rejected after shutdown" + ); + assert!(!shutdown.is_finished(), "shutdown must wait for an admitted registration"); + + drop(held_cleanup); + drop(admission); + timeout(Duration::from_secs(2), shutdown).await??; + assert_eq!(connections.cleanup_tx().capacity(), initial_capacity); + Ok(()) + } + + #[tokio::test] + async fn shutdown_with_unpolled_shared_body_leaves_no_requests_or_subscribers( + ) -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41016".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/unpolled_shutdown.ts"; + register_user(&users, addr, id.stream_uid(), "shutdown_user", url).await?; + + let handle = providers + .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) + .ok_or("provider allocation missing")?; + let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); + let pending = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + let (unpolled_body, _, _) = SharedStreamManager::register_shared_stream( + super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(managed_handle), + pending, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("register failed")?; + + assert_eq!(providers.get_provider_connections_count(), 1); + assert_eq!(users.active_streams().await.len(), 1); + + connections.shutdown().await; + + assert_eq!(providers.get_provider_connections_count(), 0); + assert!(users.active_streams().await.is_empty()); + assert!(manager.get_shared_state(url).await.is_none()); + + drop(unpolled_body); + Ok(()) + } + + #[tokio::test] + async fn shared_cold_miss_does_not_consume_cleanup_capacity_or_emit_cleanup( + ) -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41017".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let initial_capacity = connections.cleanup_tx().capacity(); + + let ctx = super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + + let res = SharedStreamManager::subscribe_shared_stream( + ctx, + "https://example.invalid/live/non_existent.ts", + &addr, + id, + 0, + ConnectionKind::Normal, + ) + .await?; + + assert!(res.is_none()); + assert_eq!( + connections.cleanup_tx().capacity(), + initial_capacity, + "cold miss must not consume cleanup capacity" + ); + Ok(()) + } + + #[tokio::test] + async fn shared_state_removed_between_preflight_and_commit_leaves_no_claim( + ) -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41018".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/removed_between.ts"; + + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + manager.shared_streams.write().await.by_key.insert(Arc::from(url), state); + + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + if let Ok(mut lock) = manager.test_preflight_barrier.lock() { + *lock = Some(Arc::clone(&barrier)); + } + + let initial_capacity = connections.cleanup_tx().capacity(); + + let sub_task = tokio::spawn({ + let app_cfg = Arc::clone(&app_cfg); + let manager = Arc::clone(&manager); + let providers = Arc::clone(&providers); + let connections = Arc::clone(&connections); + async move { + let ctx = super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + SharedStreamManager::subscribe_shared_stream(ctx, url, &addr, id, 0, ConnectionKind::Normal).await + } + }); + + barrier.wait().await; + + { + let mut reg = manager.shared_streams.write().await; + reg.by_key.remove(url); + } + + let res = sub_task.await??; + assert!(res.is_none()); + assert_eq!(connections.cleanup_tx().capacity(), initial_capacity); + assert!(manager.get_shared_state(url).await.is_none()); + Ok(()) + } + + #[tokio::test] + async fn pending_join_keeps_origin_alive_when_last_subscriber_leaves() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (_providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41019".parse()?; + let first_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/pending_join.ts"; + + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + state.register_subscriber(first_id, &addr, CancellationToken::new()).await; + { + let mut reg = manager.shared_streams.write().await; + reg.by_key.insert(Arc::from(url), Arc::clone(&state)); + reg.key_by_subscriber.insert(first_id, Arc::from(url)); + } + + // Simulate a join that committed its registry entry but has not yet registered + // its subscriber in the origin. + let pending_join = super::PendingJoinGuard::new(&state); + assert!(state.has_pending_joins()); + + // The last (and only) subscriber leaves; the origin must not be torn down because + // the pending join is about to land on it. + manager.release_subscriber(first_id).await; + assert!(manager.get_shared_state(url).await.is_some(), "origin must survive a pending join"); + + // The join completes; the origin has no subscribers and no pending joins. + pending_join.commit(); + assert!(!state.has_pending_joins()); + assert!(state.subscribers.read().await.is_empty(), "origin has no subscribers after release"); + Ok(()) + } + + #[test] + fn uncommitted_shared_meter_registration_is_rolled_back() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + let url = "https://example.invalid/live/meter-rollback.ts"; + + let (meter_uid, pending) = manager.reserve_meter_uid(url, || 41); + assert_eq!(meter_uid, 41); + assert_eq!(manager.meter_count(), 1); + + drop(pending); + assert_eq!(manager.meter_count(), 0); + } + + #[test] + fn pending_meter_rollback_does_not_remove_adopted_entry() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + let url = "https://example.invalid/live/meter-replacement.ts"; + + // A reserved the meter; B then commits a shared origin and adopts it. + let (_, stale) = manager.reserve_meter_uid(url, || 51); + manager.adopt_meter_uid(url, 999); + + // A aborts: its pending rollback must not remove the adopted entry. + drop(stale); + assert_eq!(manager.meter_count(), 1); + assert_eq!(manager.lock_meter_uids().get(url).map(|entry| entry.uid), Some(51)); + assert_eq!(manager.lock_meter_uids().get(url).and_then(|entry| entry.owner), Some(999)); + } + + #[test] + fn successor_adoption_overwrites_stale_owner_and_survives_old_teardown() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + let url = "https://example.invalid/live/meter-successor.ts"; + + let (uid, guard) = manager.reserve_meter_uid(url, || 71); + manager.adopt_meter_uid(url, 100); + // A successor origin commits the same URL and takes over the meter. + manager.adopt_meter_uid(url, 200); + assert_eq!(manager.lock_meter_uids().get(url).and_then(|entry| entry.owner), Some(200)); + + // A stale teardown by the predecessor must not remove the successor's entry. + manager.remove_meter_uid_if_owned_by(url, Some(100)); + assert_eq!(manager.meter_count(), 1); + assert_eq!(manager.lock_meter_uids().get(url).map(|entry| entry.uid), Some(uid)); + guard.expect("reservation").commit(); + } + + #[tokio::test] + async fn shutdown_clears_shared_meter_registrations() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + + let (_, pending) = manager.reserve_meter_uid("https://example.invalid/live/meter-shutdown.ts", || 61); + pending.expect("new registration").commit(); + assert_eq!(manager.meter_count(), 1); + + manager.shutdown().await; + assert_eq!(manager.meter_count(), 0); + } } diff --git a/bin/build_docker.sh b/bin/build_docker.sh index 1ff289a6f..1fcd8c9e0 100755 --- a/bin/build_docker.sh +++ b/bin/build_docker.sh @@ -24,6 +24,7 @@ BRANCH="$1" case "$BRANCH" in master) TAG_SUFFIX="latest" ;; develop) TAG_SUFFIX="dev" ;; + experimental) TAG_SUFFIX="experimental" ;; *) echo "🧨 Error: Branch '$BRANCH' not supported"; exit 1 ;; esac @@ -130,20 +131,25 @@ REPO_OWNER_LC="${REPO_OWNER,,}" for IMAGE_NAME in "${!MULTI_PLATFORM_IMAGES[@]}"; do BUILD_TARGET="${MULTI_PLATFORM_IMAGES[$IMAGE_NAME]}" - TAG_VERSION="ghcr.io/${REPO_OWNER_LC}/${IMAGE_NAME}:${VERSION}" TAG_BRANCH="ghcr.io/${REPO_OWNER_LC}/${IMAGE_NAME}:${TAG_SUFFIX}" + if [ "$BRANCH" = "experimental" ]; then + # Experimental images must not overwrite official release version tags + TAG_VERSION="ghcr.io/${REPO_OWNER_LC}/${IMAGE_NAME}:experimental-${VERSION}" + else + TAG_VERSION="ghcr.io/${REPO_OWNER_LC}/${IMAGE_NAME}:${VERSION}" + fi echo "🎯 Building multi-platform image: ${IMAGE_NAME}" # THE FIX: Using type=gha for automatic GitHub Actions cache management. - # No more local files, no more leftover artifacts. + # Scoped per branch/tag_suffix to avoid cache pollution between develop and experimental. docker buildx build -f Dockerfile.manual \ -t "${TAG_VERSION}" \ -t "${TAG_BRANCH}" \ --target "$BUILD_TARGET" \ --platform "linux/amd64,linux/arm64" \ - --cache-from "type=gha,scope=${IMAGE_NAME}" \ - --cache-to "type=gha,mode=max,scope=${IMAGE_NAME}" \ + --cache-from "type=gha,scope=${IMAGE_NAME}-${TAG_SUFFIX}" \ + --cache-to "type=gha,mode=max,scope=${IMAGE_NAME}-${TAG_SUFFIX}" \ --push \ . done diff --git a/bin/run-testkit-scenarios.sh b/bin/run-testkit-scenarios.sh new file mode 100755 index 000000000..619cd104f --- /dev/null +++ b/bin/run-testkit-scenarios.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd -P)" +cd "${REPO_ROOT}" + +TARGET_DIR="${CARGO_TARGET_DIR:-${REPO_ROOT}/target}" +SUT_BINARY="${TARGET_DIR}/debug/tuliprox" +TESTKIT_BINARY="${TARGET_DIR}/debug/tuliprox-testkit" +REPORT_DIR="${REPO_ROOT}/testkit-report" + +if [[ ! -x "${SUT_BINARY}" ]]; then + echo "❌ Tuliprox SUT binary not found at ${SUT_BINARY}. Build it first: cargo build -p tuliprox" >&2 + exit 1 +fi + +if [[ ! -x "${TESTKIT_BINARY}" ]]; then + echo "❌ Testkit binary not found at ${TESTKIT_BINARY}. Build it first: cargo build -p tuliprox-testkit" >&2 + exit 1 +fi + +mkdir -p "${REPORT_DIR}" +export TULIPROX_TESTKIT_SUT_BINARY="${SUT_BINARY}" + +SUITE_RUN_ID="${TULIPROX_TESTKIT_RUN_ID:-$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || echo "suite-$(date +%s)")}" + +FAILED_SCENARIOS=() +PASSED_COUNT=0 + +echo "πŸš€ Starting testkit scenario suite (run_id: ${SUITE_RUN_ID})..." + +for scenario_path in test/fixtures/testkit/scenarios/*.yml; do + scenario_name="$(basename "${scenario_path}" .yml)" + scenario_dir="${REPORT_DIR}/${scenario_name}" + rm -rf "${scenario_dir}" + mkdir -p "${scenario_dir}" + echo " β–Ά Running scenario: ${scenario_name}" + + set +e + "${TESTKIT_BINARY}" controller --scenario "${scenario_path}" --report-directory "${scenario_dir}" --run-id "${SUITE_RUN_ID}" > "${scenario_dir}/scenario.log" 2>&1 + exit_code=$? + set -e + + if [[ ${exit_code} -eq 0 ]]; then + echo " βœ… ${scenario_name} PASSED" + PASSED_COUNT=$((PASSED_COUNT + 1)) + else + echo " ❌ ${scenario_name} FAILED (exit code ${exit_code})" + FAILED_SCENARIOS+=("${scenario_name}") + if [[ ! -f "${scenario_dir}/run.json" ]]; then + cat < "${scenario_dir}/run.json" +{ + "schema_version": 1, + "scenario": "${scenario_name}", + "outcome": "failed", + "policy_hash": "pre-start", + "run_id": "${SUITE_RUN_ID}", + "error_kind": "controller_crashed", + "error_message": "Controller exited with code ${exit_code} before run.json could be generated", + "exit_code": ${exit_code}, + "events": [] +} +EOF + cat < "${scenario_dir}/summary.txt" +scenario: ${scenario_name} +outcome: failed +policy: pre-start +run_id: ${SUITE_RUN_ID} +events: 0 +EOF + touch "${scenario_dir}/events.ndjson" + cat < "${scenario_dir}/junit.xml" + + +EOF + fi + cat "${scenario_dir}/scenario.log" >&2 + fi +done + +echo "" +echo "========================================" +echo "Testkit Scenario Summary:" +echo " Run ID: ${SUITE_RUN_ID}" +echo " Passed: ${PASSED_COUNT}" +echo " Failed: ${#FAILED_SCENARIOS[@]}" +echo "========================================" + +FAILED_JSON_ARRAY="[]" +if [[ ${#FAILED_SCENARIOS[@]} -gt 0 ]]; then + FAILED_JSON_ARRAY="[$(printf '"%s",' "${FAILED_SCENARIOS[@]}" | sed 's/,$//')]" +fi + +cat < "${REPORT_DIR}/summary.json" +{ + "run_id": "${SUITE_RUN_ID}", + "total": $((PASSED_COUNT + ${#FAILED_SCENARIOS[@]})), + "passed": ${PASSED_COUNT}, + "failed": ${#FAILED_SCENARIOS[@]}, + "failing_scenarios": ${FAILED_JSON_ARRAY} +} +EOF + +if [[ ${#FAILED_SCENARIOS[@]} -gt 0 ]]; then + echo "❌ Failing scenarios: ${FAILED_SCENARIOS[*]}" >&2 + exit 1 +fi + +echo "βœ… All scenarios passed successfully!" diff --git a/docker/Dockerfile b/docker/Dockerfile index 65d7d9455..c76e9d1a6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,15 +35,19 @@ COPY backend/parser/Cargo.toml ./backend/parser/ COPY backend/processing/Cargo.toml ./backend/processing/ COPY backend/repository/Cargo.toml ./backend/repository/ COPY backend/session/Cargo.toml ./backend/session/ +COPY tools/tuliprox-testkit/Cargo.toml ./tools/tuliprox-testkit/ -RUN mkdir -p frontend/src backend/app/src/tools && \ +RUN mkdir -p frontend/src backend/app/src/tools tools/tuliprox-testkit/src && \ printf 'fn main() {}\n' > frontend/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/tools/flags_builder.rs && \ + printf '\n' > tools/tuliprox-testkit/src/lib.rs && \ + printf 'fn main() {}\n' > tools/tuliprox-testkit/src/main.rs && \ for crate in auth btree config-loader core curation dvr hls iptv library media-server metadata messaging mpegts parser processing repository session; do \ mkdir -p "backend/${crate}/src"; \ printf '\n' > "backend/${crate}/src/lib.rs"; \ - done + done && \ + cargo metadata --no-deps --format-version 1 > /dev/null # ----------------------------------------------------------------- # Stage 1: Build the Rust binary for production diff --git a/docker/cross.Dockerfile b/docker/cross.Dockerfile index 67c82fc8e..56d099caa 100644 --- a/docker/cross.Dockerfile +++ b/docker/cross.Dockerfile @@ -39,15 +39,19 @@ COPY backend/parser/Cargo.toml ./backend/parser/ COPY backend/processing/Cargo.toml ./backend/processing/ COPY backend/repository/Cargo.toml ./backend/repository/ COPY backend/session/Cargo.toml ./backend/session/ +COPY tools/tuliprox-testkit/Cargo.toml ./tools/tuliprox-testkit/ -RUN mkdir -p frontend/src backend/app/src/tools && \ +RUN mkdir -p frontend/src backend/app/src/tools tools/tuliprox-testkit/src && \ printf 'fn main() {}\n' > frontend/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/main.rs && \ printf 'fn main() {}\n' > backend/app/src/tools/flags_builder.rs && \ + printf '\n' > tools/tuliprox-testkit/src/lib.rs && \ + printf 'fn main() {}\n' > tools/tuliprox-testkit/src/main.rs && \ for crate in auth btree config-loader core curation dvr hls iptv library media-server metadata messaging mpegts parser processing repository session; do \ mkdir -p "backend/${crate}/src"; \ printf '\n' > "backend/${crate}/src/lib.rs"; \ - done + done && \ + cargo metadata --no-deps --format-version 1 > /dev/null # ----------------------------------------------------------------- # Stage 1: Build the Rust binary for production diff --git a/docs/src/configuration/api-proxy.md b/docs/src/configuration/api-proxy.md index e32c6a1c8..e32d62bd2 100644 --- a/docs/src/configuration/api-proxy.md +++ b/docs/src/configuration/api-proxy.md @@ -126,6 +126,7 @@ in your `config.yml`. Without it, these fields are purely cosmetic! | `epg_timeshift` | String | No | `None` | Shifts EPG times for users in different time zones. Formats supported: `[-+]hh:mm` or `TimeZone`. Examples: `-2:30` (minus 2h30m), `1:45` (1h45m), `+0:15` (15m), `2` (2h), `:30` (30m), `:3` (3m), `Europe/Paris`, `America/New_York`. Only applies when `epg_url` is configured. | | `epg_request_timeshift` | String | No | `None` | Shifts EPG times for users in different time zones specifically to adjust catchup requests | | `max_connections` | Int | No | `0` | Hard limit of concurrent streams for *this* user. `0` = Unlimited. **Requires** `user_access_control: true` in `config.yml` to be enforced. | +| `soft_connections` | Int | No | `0` | Additional streams allowed *above* `max_connections`. The total concurrent allowance is `max_connections + soft_connections`, so `max_connections: 1` with `soft_connections: 1` permits two concurrent streams. Soft streams are preempted before normal ones. `0` = no extra allowance. **Requires** `user_access_control: true` in `config.yml` to be enforced. | | `status` | Enum | No | `Active` | Possible values: `Active`, `Trial`, `Expired`, `Banned`, `Disabled`, `Pending`. **Requires** `user_access_control: true` in `config.yml` to block non-active streaming. | | `exp_date` | UnixTs | No | `None` | Locks the user out after this Unix timestamp. **Requires** `user_access_control: true` in `config.yml` to be enforced. | | `ui_enabled` | Bool | No | `true` | Allows this specific user to log into the Web UI to manage their own favorites/bouquets. | diff --git a/docs/src/configuration/connection-handling-runtime-internals.md b/docs/src/configuration/connection-handling-runtime-internals.md index 782a52988..1d7629a9b 100644 --- a/docs/src/configuration/connection-handling-runtime-internals.md +++ b/docs/src/configuration/connection-handling-runtime-internals.md @@ -277,11 +277,15 @@ Playback becomes visible/active when the runtime updates connection state throug That is the point where: - logical playback +- the individual HTTP request claim - stream visibility - socket/provider tracking become externally visible as an active running stream. +The visible stream and the request claim are deliberately separate. Segment, range and reconnect requests can reuse one +logical playback without allowing completion of one body to remove its siblings. + ## 9. Cleanup paths When the stream ends or is kicked, cleanup may: @@ -294,9 +298,14 @@ When the stream ends or is kicked, cleanup may: Important cleanup behaviors: +- registration reserves a bounded cleanup permit before publishing mutable state +- the response body owns that permit and releases the exact request on EOF, error, cancellation or drop +- provider cleanup uses the request identity and binding incarnation captured during acquire instead of rediscovering + ownership from the socket address - kicked cleanup can terminate the associated session immediately - adaptive/HLS-style playback may remain preserved for a TTL - preserved does not mean counted +- server shutdown closes new admission, drains active state directly and joins its owned workers ## 10. Background probe and resolve tasks diff --git a/docs/src/configuration/connection-handling-session-implementation-notes.md b/docs/src/configuration/connection-handling-session-implementation-notes.md index 023c9fcea..ea9a72473 100644 --- a/docs/src/configuration/connection-handling-session-implementation-notes.md +++ b/docs/src/configuration/connection-handling-session-implementation-notes.md @@ -46,7 +46,8 @@ It is stable for follow-up requests of the same playback, but it is not always d - plain TS live uses a socket-scoped playback token - VOD, series, catchup, and local playback use a logical token suitable for reopen/seek/range workflows -- HLS/DASH playlist starts use a token scoped to that initial adaptive playback start, and rewritten segment URLs carry that token forward +- HLS playlist starts use a token scoped to that initial adaptive playback start, and rewritten segment URLs carry that token forward +- DASH playlist starts use an equally scoped token, but the redirected player fetches DASH segments directly from the provider It is used to answer questions like: @@ -117,6 +118,20 @@ Why: This means the same provider-affine HLS, VOD, series, or catchup stream must keep the same provider account for its follow-up requests, even though those requests may arrive on different sockets. +### DASH delivery boundary (redirect only) + +Tuliprox treats DASH as **redirect-only**: + +- DASH requests (`PlaylistItemType::LiveDash` or requests with `.mpd` extension) always resolve to an HTTP redirect directly to + the upstream provider URL. +- DASH reverse-proxying and segment caching are **not supported**. Enabling reverse-proxy mode for a user or target does not route + DASH streams through the HLS proxy handler or cache segments in Tuliprox. +- DASH provides provider-account affinity only when it reuses an existing pinned session. Direct segment fetching does not + activate or renew an Active lease, and Tuliprox does not deliver DASH media through the proxy. +- Because segments are fetched directly between the player and the upstream provider, Tuliprox cannot count individual segment + requests or confirm first-byte delivery for DASH segments. Full segment lease guarantees apply only to streams proxied + directly through Tuliprox (TS, HLS, VOD, Series, Catchup). + ### Adaptive preserved session Adaptive live playback (`LiveHls`, `LiveDash`) can stay logically alive for a short TTL even after the current socket disconnects. @@ -127,22 +142,98 @@ That logic lives in `ActiveUserManager` and uses: - `build_preserved_stream_expiry(...)` - `process_due_adaptive_expiry_entries(...)` -### Provider reservation +### Provider slot lease -Provider reservation is a provider-slot affinity mechanism, not a user admission mechanism. +A provider reservation is now a *slot lease* owned by a playback identity, not by a socket. It is a provider-slot +affinity mechanism, not a user admission mechanism. One owner holds at most one lease, so repeated requests of the same +playback reuse the slot instead of stacking a reservation per attempt. -Current TTL mapping: +A lease has three states: + +- `Starting`: claimed when a provider allocation is acquired. It pins the provider for its own playback and blocks + nobody else. It expires after a short startup deadline regardless of the configured TTL, so an abandoned start or a + manifest-retry loop never accumulates reservations. +- `Active`: reached only when real media delivery is confirmed β€” the first provider media byte forwarded to a client as + an `OK`/`206` media response. Manifest, HEAD, key, map and error fetches never confirm a lease. Only an `Active` lease + with a configured reconnect window reserves capacity against other playbacks. +- `Idle`: a confirmed, reconnect-capable playback that ended cleanly keeps its slot for the reconnect window so it can + reattach to the same provider. + +TTL mapping (the reconnect window): - HLS/DASH use `hls_session_ttl_secs` - Catchup uses `catchup_session_ttl_secs` - other item types use `0` -This affects provider reuse between short request gaps, not whether the session should use session admission. +Lease end is outcome-driven: provider failure, preemption, kick and timeout release capacity immediately; a clean finish +of a reconnect-capable playback keeps the idle window. Important: - VOD/movie/series still have strict provider affinity on follow-up requests -- they just do not currently get an extra post-request reservation TTL like HLS/Catchup do +- they just do not currently get an extra post-request reconnect window like HLS/Catchup do +- An HTTP 200 manifest response is not playback and does not confirm a lease; neither do HEAD, key, map or error-video + responses. Only an `OK`/`206` media response confirms media delivery, which is what previously let entry + reservations block unrelated clients behind the same reverse proxy + +### Multi-user socket isolation and allocation-based provider connections + +When multiple users connect through the same reverse proxy socket: + +- `ActiveUserManager` tracks user sessions per user key and stream UID. `SocketRegistration` records all registered + usernames for each `SocketAddr`, ensuring concurrent users on the same socket never overwrite or prematurely evict + each other's session state. +- `ActiveProviderManager` indexes single connections by unique `AllocationId` rather than grouping them under a single + socket address key (`ClientConnectionId`). Secondary socket indexes exist only for whole-transport close and kick + signals. +- Stream details and QoS fallbacks are updated directly by `stream_uid` (`update_stream_detail_by_uid`) rather than + relying on ambiguous socket lookups. + +### Request ownership and binding generations + +The runtime uses separate identities for separate ownership scopes: + +| Identity | Scope | +| :--- | :--- | +| session token | one logical user playback | +| `PlaybackLeaseId` | one logical provider-capacity lease | +| `PlaybackRequestId` | one provider request attached to that lease | +| `ProviderBindingTag` | one generation of the owner's provider-account binding | +| `SharedSubscriberId` | one viewer attached to a shared MPEG-TS origin | + +Parallel range, segment and reconnect requests may belong to the same logical stream while retaining independent request +claims. Completing one request removes only that claim. The logical stream and provider lease remain until their own +terminal conditions are met. + +A provider binding increments its generation when the same playback moves to a different provider or reactivates an +idle lease. Cleanup created for an older generation cannot clear the successor binding. This protects new playback from +delayed body drops, provider errors and asynchronous HLS cleanup belonging to the preceding request. + +### Cleanup ownership and overload behavior + +Connection registration reserves a cleanup permit before publishing mutable user or provider state. Ownership then +moves into the response body or shared-subscriber body. Dropping an unpolled body, cancelling an await, reaching EOF or +reporting an error all converge on request-specific, idempotent cleanup. + +The regular cleanup queue is bounded by `reverse_proxy.stream.cleanup_queue_capacity`. Admission waits for a bounded +interval when all permits are held and then rejects the request instead of registering a stream without guaranteed +cleanup. HLS origin I/O uses a separate bounded control lane that is preferred over ordinary body cleanup, preventing a +saturated subscriber workload from starving release of an origin provider handle. + +Graceful shutdown first closes the admission gate, then removes shared origins and active user claims, releases provider +leases, flushes stream history and cancels and joins the owned background workers. Cleanup emitted by a later body drop +is harmless because the terminal indices have already been emptied. + +### Shared MPEG-TS subscriber isolation + +A shared origin is keyed by stream URL, but its viewers are keyed by `SharedSubscriberId`, never by `SocketAddr`. This +allows several users behind one reverse proxy connection to consume independent response bodies without replacing one +another. + +Each subscriber has both a chunk limit and a byte semaphore for queued payload. Burst replay and live delivery use a +progress deadline that is renewed only after a successful send. Timeout, cancellation and a closed receiver are distinct +terminal outcomes; an incomplete replay never advances directly to the live tail. A pending-join guard also prevents +the last existing subscriber from tearing down an origin while another subscriber is committing. ## Where session tokens come from @@ -278,11 +369,13 @@ Main entry point: - `ActiveUserManager::update_connection(...)` -This creates or reuses the tracked logical stream entry. +This creates or reuses the tracked logical stream entry and attaches a request claim. Important behavior: - same `session_token` reuses the logical stream +- every response receives its own request UID even when it reuses that stream +- ending one response detaches only its request claim; the stream is removed when its final claim ends - stream metrics and duration stay tied to the logical session - for non-socket-bound sessions, the session remembers multiple active addresses @@ -341,6 +434,10 @@ For non-socket-bound sessions: This is the key behavior that keeps VOD and local playback stable across multiple sockets. +Queued provider cleanup carries the `PlaybackRequestId` and owner captured during acquire, while provider handles retain +their `ProviderBindingTag`. The lease table validates both against the current binding, so cleanup does not need to +rediscover ownership from a possibly reused socket address. + ## URL persistence rules Session URL handling is intentionally different by stream type. @@ -365,14 +462,16 @@ For live playback, following the redirected provider URL is still acceptable and Read these files together before changing session logic: -- `backend/app/src/api/api_utils.rs` -- `backend/app/src/api/endpoints/m3u_api.rs` -- `backend/app/src/api/endpoints/xtream_api.rs` -- `backend/app/src/api/endpoints/hls_api.rs` -- `backend/app/src/api/model/active_user_manager.rs` -- `backend/app/src/api/model/active_provider_manager.rs` -- `backend/app/src/api/model/metadata_update_manager.rs` -- `shared/src/model/playlist.rs` +- `backend/core/src/model/playback.rs` +- `backend/session/src/provider_leases.rs` +- `backend/session/src/active_user_manager/mod.rs` +- `backend/session/src/active_provider_manager.rs` +- `backend/session/src/connection_manager.rs` +- `backend/session/src/streams/shared_stream_manager.rs` +- `backend/app/src/api/api_utils/mod.rs` +- `backend/app/src/api/model/streams/active_client_stream.rs` +- `backend/app/src/api/endpoints/hls_api/session.rs` +- `backend/hls/src/origin.rs` ## Tests that should stay green @@ -387,6 +486,12 @@ These tests cover the most fragile parts of the current logic: - `recently_evicted_vod_uses_session_reentry_guard` - `update_session_addr_prunes_previous_registration_for_socket_bound_session` - `test_adaptive_session_release_connection_preserves_logical_stream_and_start_time` +- `stale_direct_cleanup_after_rebind_preserves_successor_request` +- `subscribers_on_same_socket_do_not_replace_each_other` +- `send_client_chunk_respects_byte_budget` +- `control_cleanup_lane_precedes_a_ready_body_cleanup_backlog` +- `shutdown_with_all_cleanup_permits_held_completes` +- `http_provider_lease_downstream_smoke_test` If you change session code and one of these assumptions no longer holds, update the documentation and the tests in the same change. diff --git a/docs/src/configuration/reverse-proxy.md b/docs/src/configuration/reverse-proxy.md index ef43516b9..8e0a18f26 100644 --- a/docs/src/configuration/reverse-proxy.md +++ b/docs/src/configuration/reverse-proxy.md @@ -95,6 +95,18 @@ Examples: * valid: `evict_user_same_ip_latest`, then `evict_user_latest` * invalid: `evict_user_latest`, then `evict_user_same_ip_latest` +Several configuration details are easy to misread: + +* An omitted `admission_strategies` key is **not** the same as an empty list. When the key is absent + and `grace_period_millis > 0` (default `2000`), Tuliprox falls back to a single grace strategy selected + by `grace_period_hold_stream` (`grace_hold_stream` when `true`, otherwise `grace_instant_stream`). + Use `admission_strategies: []` to disable that fallback. +* `admission_strategies: []` disables only the user-admission fallback. Provider-side grace is still + governed by `grace_period_millis`; set it to `0` if you want no grace or stream holding at all. +* A grace strategy listed before an eviction strategy is resolved first, but it does not make the later + rules unreachable. If grace cannot be granted, the remaining strategies, including eviction, are still + evaluated. + ```yaml reverse_proxy: stream: @@ -109,6 +121,8 @@ reverse_proxy: hls_session_ttl_secs: 15 catchup_session_ttl_secs: 45 shared_burst_buffer_mb: 12 + cleanup_queue_capacity: 4096 + recent_eviction_reentry_ttl_ms: 3000 metrics_enabled: false ``` @@ -129,6 +143,8 @@ reverse_proxy: | `hls_session_ttl_secs` | Int | `15` | Keeps virtual provider slot open between HLS segment (`.ts`) requests to prevent provider bans for "Account Hopping". | | `catchup_session_ttl_secs` | Int | `45` | Same session-holding principle applied to Archive/Catchup TV. See notes on section [Session TTLs for HLS & Catchup](#session-ttls-for-hls-m3u8--catchup) for details. | | `shared_burst_buffer_mb` | Int | `12` | Minimum burst buffer size (in MB) used for shared live streams to immediately synchronize new clients without Keyframe dropouts. See notes on section [Shared Live Streams](#shared-live-streams) for details. | +| `cleanup_queue_capacity` | Int | `4096` | Maximum number of concurrent cleanup permits available to active response bodies and shared subscribers. Must be at least `1`. If all permits remain held, new stream admission waits for a bounded interval and then returns `503 Service Unavailable` instead of growing memory without limit. | +| `recent_eviction_reentry_ttl_ms` | Int | `3000` | Time window after an eviction during which a retry of the evicted playback must not evict its replacement. The guard is scoped to the user, client address and channel, so unrelated clients are not blocked. Suppressed retries end quietly (no `user_connections_exhausted` video and no `ConnectionDenied` event), since they are not a real connection-limit refusal. Raise this for players with slow automatic retries; `0` disables the guard. The cumulative suppression count is exposed as `reentry_suppressed_total` in the server status (`GET /api/v1/status`). | ### 1.1 `retry` & `buffer` (Deep Dive) @@ -148,6 +164,10 @@ Stream-type provider behavior: reopen requests must stay on the provider account pinned in that session; if that account is unavailable, Tuliprox fails the follow-up instead of silently migrating it. * `retry: false` disables stream-open retry/failover for stream requests. +* **DASH delivery scope:** DASH streams (`LiveDash`, `.mpd`) are delivered exclusively via HTTP redirect to the upstream + provider. Tuliprox does not reverse-proxy or cache DASH segments. Even when reverse-proxy mode is active, DASH requests + are not routed into the HLS proxy pipeline. DASH sessions retain provider-account affinity during reopen/refresh within + `hls_session_ttl_secs`. #### Ring-Buffer Calculation @@ -192,8 +212,23 @@ to maintain account affinity: Important boundary: * These TTLs are for HLS/catchup style session continuity. -* Regular TS/VOD/local playback is socket-bound for admission and user-connection counting. -* Opening the same TS/VOD stream on a second socket consumes another connection or soft slot. +* Plain TS live playback is socket-bound. A new TS socket is a separate playback attempt. +* VOD, series and local playback use logical session identities so size, range, seek and reopen requests can continue the + same playback across sockets. A genuinely separate playback still consumes another connection or soft slot. +* A provider reservation becomes capacity-relevant only after Tuliprox forwards real media bytes. Manifests, HEAD + requests, keys, maps and error responses do not confirm a provider slot lease. + +### 1.5 Cleanup admission and overload behavior + +Every admitted streaming response must own a terminal cleanup permit before it can publish user or provider state. The +permit follows the response body and releases the exact request when the body completes, is dropped without being +polled, or fails while streaming. Shared subscribers use the same ownership rule without reserving a second permit for +the same body. + +`cleanup_queue_capacity` bounds how many of these cleanup owners may exist at once. If the queue remains saturated for +the admission deadline, Tuliprox rejects the new request with `503 Service Unavailable`; it does not admit an +unprotected stream. HLS origin-control cleanup has reserved capacity so ordinary body churn cannot prevent a provider +handle from being released. Changing the setting requires a server restart. --- @@ -541,14 +576,14 @@ reverse_proxy: HLS streams do not consist of an endless TCP pipe. Instead, the player downloads small `.ts` segments every few seconds (e.g., `seg1.ts`, `seg2.ts`). -If Tuliprox released and re-acquired the provider slot for every single segment, -providers would block the account for "Account Hopping" or spam. Tuliprox simulates a continuous session: +If Tuliprox selected a different provider account for every segment, providers could treat the traffic as account +hopping. Tuliprox therefore keeps a logical, confirmed slot lease for the playback: -* `hls_session_ttl_secs: 15`: After a `.ts` segment finishes downloading, the physical slot to the provider is closed, - but the "Virtual Slot" for this specific user remains reserved for 15 seconds. No other user can steal this slot - during this window. Channel switches from the same client can immediately take over the reservation. -* The same principle applies to Archive/Catchup TV (`catchup_session_ttl_secs: 45`), - which shares the same fragmentation and seeking issues. +* `hls_session_ttl_secs: 15`: After Tuliprox has forwarded actual media bytes, a cleanly finished HLS request may keep + its provider account reserved for this playback for up to 15 seconds. A manifest-only or abandoned start does not + create a capacity reservation. +* Archive/Catchup playback uses the same confirmed-lease principle with `catchup_session_ttl_secs: 45`. +* Provider errors, preemption, kicks and timeouts release the lease immediately rather than keeping the reconnect window. --- @@ -573,6 +608,12 @@ To ensure a user who tunes in 10 seconds later doesn't get player errors due to Tuliprox continuously keeps the last X Megabytes (`shared_burst_buffer_mb`, default `12`) in RAM. It fires this burst buffer at new subscribers so their decoders can instantly synchronize. +Each viewer has a distinct subscriber identity even when several viewers arrive through the same reverse-proxy socket. +The per-subscriber delivery queue is bounded by both chunk count and retained bytes. A client that stops making progress +is disconnected after `shared_subscriber_idle_timeout_secs`; it cannot indefinitely retain the burst buffer or stall +other viewers. If initial burst replay cannot complete, that subscriber ends instead of silently skipping missing data +and continuing with the live tail. + --- ### The "VLC Seek Problem" & Grace Periods diff --git a/docs/src/operations-debugging.md b/docs/src/operations-debugging.md index 4dfc74240..9cee01763 100644 --- a/docs/src/operations-debugging.md +++ b/docs/src/operations-debugging.md @@ -417,3 +417,88 @@ If hot-reload fails to trigger, a container restart (`docker restart tuliprox`) is the safest fallback. --- + +## 8. Runtime Liveness Watchdog + +A reverse proxy can wedge without crashing: the process is still alive, but the async scheduler stops making progress and +the logs simply stop. At the OS level this looks like an idle process, so `docker stats` and `ps` cannot tell it apart +from a healthy one. + +The watchdog detects exactly that. It is **opt-in and off by default**. + +### What it does + +* **Heartbeat (inside the runtime):** a small task records a monotonic timestamp every second and measures how late it + was scheduled. +* **Watchdog thread (outside the runtime):** a separate `std::thread` watches the heartbeat age. If the scheduler is + wedged, the heartbeat stops advancing and the thread logs a diagnostic snapshot. +* **On a stall** it logs (once, then every 30 s while it continues) the heartbeat age, runtime metrics (worker count, + alive tasks, global queue depth, per-worker park counts) and, on Linux, a `/proc/self/task` inventory with each + thread's id, name, state and waiting channel. +* **On recovery** it logs that the heartbeat is back. + +The watchdog itself only reports. In mode `2` it can additionally exit the process so a supervisor restarts it (see below). + +### Modes + +`TULIPROX_WATCHDOG` is a mode selector, not a boolean: + +| Value | Mode | Behaviour | +|:------------------------------------|:--------|:-----------------------------------------------------------------------------------| +| unset, `0`, anything else | off | No heartbeat task, no watchdog thread. | +| `1`, `true`, `on`, `yes`, `enabled` | observe | Detect and log stalls; never touches the process. | +| `2`, `restart` | restart | Like observe, and exit the process after the stall persists past the grace period. | + +In restart mode the process exits with code `75` (non-zero, so both `restart: unless-stopped` and `restart: on-failure` +bring it back). This is a deliberate crash-and-restart, not an in-process recovery. Only enable it if a supervisor is +actually configured; otherwise the container will stay down. + +### Settings + +| Variable | Default | Purpose | +|:-------------------------------------|:----------------|:------------------------------------------------------------------------------| +| `TULIPROX_WATCHDOG` | *(unset = off)* | Mode selector: `1` = observe, `2` = observe and restart on a confirmed stall. | +| `TULIPROX_WATCHDOG_HEARTBEAT_MS` | `1000` | Heartbeat interval. | +| `TULIPROX_WATCHDOG_STALL_MS` | `10000` | Heartbeat age above which a stall is reported. | +| `TULIPROX_WATCHDOG_RELOG_MS` | `30000` | Re-log interval while a stall continues. | +| `TULIPROX_WATCHDOG_RESTART_GRACE_MS` | `30000` | Mode `2` only: how long the stall must persist before the process exits. | + +Docker example: + +```yaml +services: + tuliprox: + restart: unless-stopped + environment: + # observe only + - TULIPROX_WATCHDOG=1 + # ...or observe and restart on a confirmed stall (needs restart policy above) + # - TULIPROX_WATCHDOG=2 +``` + +### Healthcheck + +While the watchdog runs, `GET /healthcheck` gains a `runtime` object: + +```json +{ + "status": "ok", + "runtime": { + "status": "alive", + "heartbeat_age_ms": 12, + "heartbeat_interval_ms": 1000, + "stall_threshold_ms": 10000, + "ticks": 8401, + "stall_episodes": 0, + "max_schedule_delay_ms": 3, + "uptime_ms": 8401000, + "restart_on_stall": false + } +} +``` + +`restart_on_stall` is `true` only in mode `2`, so an operator can see whether the process is armed to restart itself. + +`runtime.status` is `stalled` once the heartbeat stops. The top-level `status` intentionally stays `ok` for as long as the +HTTP server answers, so no orchestrator restarts the process. If the whole runtime is wedged, the endpoint cannot answer +at all β€” in that case the watchdog log is the signal. diff --git a/docs/src/troubleshooting.md b/docs/src/troubleshooting.md index c8229102a..f9488e948 100644 --- a/docs/src/troubleshooting.md +++ b/docs/src/troubleshooting.md @@ -205,3 +205,37 @@ any operation requiring a new descriptor (such as opening a file or creating a s ``` --- + +## 5. Server Stops Responding While the Container Keeps Running + +**The Problem:** After running normally for a while (often hours), Tuliprox stops processing and logging. The container +still shows `running` with `ExitCode: 0` and low CPU/memory; there is no panic, no shutdown message and no final log +entry. Only a manual restart recovers it. + +**Root Cause:** A wedged async runtime. Every Tokio worker is parked on a futex and the I/O driver sits in `epoll_wait`, +which the OS cannot distinguish from an idle process. This is consistent with a task holding a shared lock while waiting +forever on a network operation that has no timeout (for example an upstream that accepts the connection but never sends a +response). + +**How to confirm:** Enable the runtime liveness watchdog (off by default): + +```yaml +services: + tuliprox: + environment: + - TULIPROX_WATCHDOG=1 # observe only + # - TULIPROX_WATCHDOG=2 # observe and restart on a confirmed stall +``` + +When the runtime stops making progress, the watchdog logs a `Runtime liveness stall` line with runtime metrics and a +per-thread `/proc/self/task` inventory, and `GET /healthcheck` reports `runtime.status: stalled`. + +In mode `1` the watchdog only reports. In mode `2` it exits the process (code `75`) once the stall has persisted past +the grace period (`TULIPROX_WATCHDOG_RESTART_GRACE_MS`, default 30 s), so a configured `restart: unless-stopped` +brings the container back automatically. Use mode `1` while you are still investigating a cause, and mode `2` once a +restart is the acceptable recovery. + +**Immediate mitigation:** restart the container. The watchdog log is what tells you the stall happened on its own and +captures what every thread was doing at that moment. + +--- diff --git a/frontend/src/app/components/country.rs b/frontend/src/app/components/country.rs index 596215630..e6ba8428c 100644 --- a/frontend/src/app/components/country.rs +++ b/frontend/src/app/components/country.rs @@ -30,7 +30,7 @@ pub fn Country(props: &CountryProps) -> Html { if let Some(svg) = flag_svg.as_ref() { diff --git a/frontend/src/app/components/dashboard/version_action_card.rs b/frontend/src/app/components/dashboard/version_action_card.rs index 48f3c006d..c565aed76 100644 --- a/frontend/src/app/components/dashboard/version_action_card.rs +++ b/frontend/src/app/components/dashboard/version_action_card.rs @@ -19,8 +19,10 @@ pub fn VersionActionCard(props: &VersionActionProps) -> Html { let services = use_service_context(); let handle_url = { + let services = services.clone(); Callback::from(move |_| { - let _ = window().open_with_url_and_target("https://github.com/euzu/tuliprox/tags", "_blank"); + let releases_link = services.config.ui_config.releases.clone(); + let _ = window().open_with_url_and_target(releases_link.as_ref(), "_blank"); }) }; diff --git a/frontend/src/app/components/home.rs b/frontend/src/app/components/home.rs index e91751dbf..eaeb038e9 100644 --- a/frontend/src/app/components/home.rs +++ b/frontend/src/app/components/home.rs @@ -47,6 +47,7 @@ struct HomeViewAccess { can_read_system_status: bool, can_read_config: bool, can_read_users: bool, + can_write_users: bool, can_read_sources: bool, can_write_playlist: bool, can_read_playlist: bool, @@ -86,12 +87,13 @@ fn is_allowed_home_view(view: ViewType, access: HomeViewAccess) -> bool { ViewType::Stats | ViewType::StreamHistory => access.can_read_system_status, ViewType::Streams => access.show_streams_page && access.can_read_system_status, ViewType::Downloads => access.can_read_downloads, - ViewType::Users => access.can_read_users, + ViewType::Users => access.can_read_users || access.can_write_users, ViewType::Plans => access.can_read_config, ViewType::Config => access.can_read_config, ViewType::SourceEditor => access.can_read_sources, ViewType::PlaylistUpdate => access.can_write_playlist, - ViewType::PlaylistSettings | ViewType::PlaylistExplorer => access.can_read_playlist, + ViewType::PlaylistSettings => access.can_read_sources || access.can_read_playlist, + ViewType::PlaylistExplorer => access.can_read_playlist, ViewType::PlaylistEpg => access.can_read_epg, ViewType::Rbac => access.is_admin, ViewType::RecordingLibrary | ViewType::RecordingRules | ViewType::RecordingRuleForm => { @@ -243,6 +245,7 @@ pub fn Home() -> Html { let can_read_system_status = services.auth.has_permission(Permission::SystemRead); let can_read_config = services.auth.has_permission(Permission::ConfigRead); let can_read_users = services.auth.has_permission(Permission::UserRead); + let can_write_users = services.auth.has_permission(Permission::UserWrite); let can_read_sources = services.auth.has_permission(Permission::SourceRead); let can_write_playlist = services.auth.has_permission(Permission::PlaylistWrite); let can_read_playlist = services.auth.has_permission(Permission::PlaylistRead); @@ -308,6 +311,7 @@ pub fn Home() -> Html { can_read_system_status, can_read_config, can_read_users, + can_write_users, can_read_sources, can_write_playlist, can_read_playlist, @@ -588,7 +592,7 @@ pub fn Home() -> Html { })} - { html_if!(can_read_users, { + { html_if!(can_read_users || can_write_users, { @@ -616,19 +620,19 @@ pub fn Home() -> Html { })} - { html_if!(can_read_playlist, { - <> + { html_if!(can_read_sources || can_read_playlist, { + })} + { html_if!(can_read_playlist, { - })} { html_if!(can_read_epg, { @@ -684,6 +688,7 @@ mod tests { can_read_system_status: true, can_read_config: true, can_read_users: true, + can_write_users: true, can_read_sources: true, can_write_playlist: true, can_read_playlist: true, @@ -729,6 +734,7 @@ mod tests { ViewType::Dashboard, super::HomeViewAccess { can_read_users: false, + can_write_users: false, can_read_config: false, can_read_sources: false, can_write_playlist: false, @@ -752,6 +758,7 @@ mod tests { super::HomeViewAccess { show_streams_page: false, can_read_users: false, + can_write_users: false, can_read_config: false, can_read_sources: false, can_write_playlist: false, diff --git a/frontend/src/app/components/playlist/input_update_card.rs b/frontend/src/app/components/playlist/input_update_card.rs index 9541e5cb8..518ec6d9f 100644 --- a/frontend/src/app/components/playlist/input_update_card.rs +++ b/frontend/src/app/components/playlist/input_update_card.rs @@ -2309,6 +2309,14 @@ mod tests { "/MESSAGES/PLAYLIST_UPDATE/CONFIGURED_COUNT", "/MESSAGES/PLAYLIST_UPDATE/DATA_UNAVAILABLE", "/MESSAGES/PLAYLIST_UPDATE/DECISION", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_FORCED_UPDATE_REQUESTED", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_INPUT_COMPLETED_PARTIALLY", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_INPUT_COMPLETED_SUCCESSFULLY", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_INPUT_FAILED", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_LIBRARY_RESCAN_COMPLETED", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_LIBRARY_RESCAN_FAILED", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_LIBRARY_RESCAN_STARTED", + "/MESSAGES/PLAYLIST_UPDATE/DETAIL_LIBRARY_SCAN_RESULT", "/MESSAGES/PLAYLIST_UPDATE/IDENTICAL_POPULATION", "/MESSAGES/PLAYLIST_UPDATE/MODE", "/MESSAGES/PLAYLIST_UPDATE/QUALITY_ACHIEVED", diff --git a/frontend/src/app/components/playlist/list.rs b/frontend/src/app/components/playlist/list.rs index 2ae37b8be..24a047696 100644 --- a/frontend/src/app/components/playlist/list.rs +++ b/frontend/src/app/components/playlist/list.rs @@ -14,9 +14,15 @@ pub fn PlaylistList() -> Html { <> { for data.iter().map(|(inputs, targets)| html! {
- {translate.t("LABEL.INPUTS")}})}> - - + { if inputs.is_empty() { + html! {} + } else { + html! { + {translate.t("LABEL.INPUTS")}})}> + + + } + } } {translate.t("LABEL.TARGETS")}
diff --git a/frontend/src/app/components/sidebar.rs b/frontend/src/app/components/sidebar.rs index 7888cd7ad..fd6cc072b 100644 --- a/frontend/src/app/components/sidebar.rs +++ b/frontend/src/app/components/sidebar.rs @@ -208,7 +208,13 @@ pub fn Sidebar(props: &SidebarProps) -> Html { })} {html_if!( - auth.has_any_permissions(Permission::ConfigRead | Permission::SourceRead | Permission::UserRead), + auth.has_any_permissions( + Permission::ConfigRead + | Permission::SourceRead + | Permission::UserRead + | Permission::UserWrite + | Permission::PlaylistRead, + ), { {html_if!(auth.is_admin(), { @@ -217,17 +223,17 @@ pub fn Sidebar(props: &SidebarProps) -> Html { {html_if!(auth.has_permission(Permission::ConfigRead), { })} - {html_if!(auth.has_permission(Permission::UserRead), { + {html_if!(auth.has_any_permissions(Permission::UserRead | Permission::UserWrite), { })} {html_if!(auth.has_permission(Permission::ConfigRead), { })} {html_if!(auth.has_permission(Permission::SourceRead), { - <> + })} + {html_if!(auth.has_any_permissions(Permission::SourceRead | Permission::PlaylistRead), { - })} } @@ -281,7 +287,13 @@ pub fn Sidebar(props: &SidebarProps) -> Html { })} {html_if!( - auth.has_any_permissions(Permission::ConfigRead | Permission::SourceRead | Permission::UserRead), + auth.has_any_permissions( + Permission::ConfigRead + | Permission::SourceRead + | Permission::UserRead + | Permission::UserWrite + | Permission::PlaylistRead, + ), { } @@ -292,17 +304,17 @@ pub fn Sidebar(props: &SidebarProps) -> Html { {html_if!(auth.has_permission(Permission::ConfigRead), { })} - {html_if!(auth.has_permission(Permission::UserRead), { + {html_if!(auth.has_any_permissions(Permission::UserRead | Permission::UserWrite), { })} {html_if!(auth.has_permission(Permission::ConfigRead), { })} {html_if!(auth.has_permission(Permission::SourceRead), { - <> + })} + {html_if!(auth.has_any_permissions(Permission::SourceRead | Permission::PlaylistRead), { - })} {html_if!( auth.has_any_permissions(Permission::PlaylistRead | Permission::PlaylistWrite | Permission::EpgRead), diff --git a/frontend/src/hooks/use_server_status.rs b/frontend/src/hooks/use_server_status.rs index 37e3af6fd..ec816c5f9 100644 --- a/frontend/src/hooks/use_server_status.rs +++ b/frontend/src/hooks/use_server_status.rs @@ -105,10 +105,7 @@ fn prune_zapped_preserved_streams(streams: &mut Vec) { return true; } let user_ip = (stream.username.clone(), stream.client_ip.clone()); - if !users_with_active.contains(&user_ip) { - return true; - } - false + !users_with_active.contains(&user_ip) }); } diff --git a/frontend/src/model/web_config.rs b/frontend/src/model/web_config.rs index f755c2da3..37f87ef81 100644 --- a/frontend/src/model/web_config.rs +++ b/frontend/src/model/web_config.rs @@ -24,6 +24,8 @@ pub struct WebConfig { #[serde(default)] pub discord: String, #[serde(default)] + pub releases: String, + #[serde(default)] pub github: String, #[serde(default = "default_documentation_url")] pub documentation: String, diff --git a/frontend/src/services/config_service.rs b/frontend/src/services/config_service.rs index 3e010aea4..203c70eb2 100644 --- a/frontend/src/services/config_service.rs +++ b/frontend/src/services/config_service.rs @@ -131,9 +131,16 @@ impl ConfigService { return; } - let can_read_config = - self.auth.has_any_permissions(Permission::ConfigRead | Permission::SourceRead | Permission::UserRead); - let can_read_api_proxy = self.auth.has_any_permissions(Permission::ConfigRead | Permission::UserRead); + let can_read_config = self.auth.has_any_permissions( + Permission::ConfigRead + | Permission::SourceRead + | Permission::UserRead + | Permission::UserWrite + | Permission::PlaylistRead + | Permission::PlaylistWrite, + ); + let can_read_api_proxy = + self.auth.has_any_permissions(Permission::ConfigRead | Permission::UserRead | Permission::UserWrite); let config_response = if can_read_config { Some( request_get_meta::( diff --git a/shared/src/defaults/network.rs b/shared/src/defaults/network.rs index a83da7769..051ad3e49 100644 --- a/shared/src/defaults/network.rs +++ b/shared/src/defaults/network.rs @@ -1,6 +1,11 @@ //! Generic networking defaults: resolve/probe delays, grace periods, retry, //! interner GC, port, user-agent, background flags. +/// Default reentry cooldown after an eviction: how long the evicted playback's +/// immediate retry is suppressed so it cannot immediately evict its replacement. +/// This is the single source of truth for the default. +pub const DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS: u64 = 3000; + default_eq_fns!( default_resolve_delay_secs, is_default_resolve_delay_secs, u16, 2; default_probe_delay_secs, is_default_probe_delay_secs, u16, 2; @@ -17,6 +22,8 @@ default_eq_fns!( default_interner_gc_min_pool_size, is_default_interner_gc_min_pool_size, u32, 100; default_custom_stream_response_error_status, is_default_custom_stream_response_error_status, u16, 502; default_event_channel_capacity, is_default_event_channel_capacity, u32, 256; + default_cleanup_queue_capacity, is_default_cleanup_queue_capacity, usize, 4096; + default_recent_eviction_reentry_ttl_ms, is_default_recent_eviction_reentry_ttl_ms, u64, DEFAULT_RECENT_EVICTION_REENTRY_TTL_MS; ); pub const fn default_resource_retry_backoff_multiplier() -> f64 { 1.0 } diff --git a/shared/src/model/config/stream.rs b/shared/src/model/config/stream.rs index 7f5a8a052..664336606 100644 --- a/shared/src/model/config/stream.rs +++ b/shared/src/model/config/stream.rs @@ -1,10 +1,11 @@ use crate::{ defaults::{ - default_as_true, default_catchup_session_ttl_secs, default_grace_period_millis, - default_grace_period_timeout_secs, default_hls_session_ttl_secs, default_shared_burst_buffer_mb, - default_shared_subscriber_idle_timeout_secs, default_stream_buffer_max_bytes_mb, - is_default_catchup_session_ttl_secs, is_default_grace_period_millis, is_default_grace_period_timeout_secs, - is_default_hls_session_ttl_secs, is_default_shared_burst_buffer_mb, + default_as_true, default_catchup_session_ttl_secs, default_cleanup_queue_capacity, default_grace_period_millis, + default_grace_period_timeout_secs, default_hls_session_ttl_secs, default_recent_eviction_reentry_ttl_ms, + default_shared_burst_buffer_mb, default_shared_subscriber_idle_timeout_secs, + default_stream_buffer_max_bytes_mb, is_default_catchup_session_ttl_secs, is_default_cleanup_queue_capacity, + is_default_grace_period_millis, is_default_grace_period_timeout_secs, is_default_hls_session_ttl_secs, + is_default_recent_eviction_reentry_ttl_ms, is_default_shared_burst_buffer_mb, is_default_shared_subscriber_idle_timeout_secs, is_default_stream_buffer_max_bytes_mb, is_false, is_true, }, error::TuliproxError, @@ -112,6 +113,16 @@ pub struct StreamConfigDto { skip_serializing_if = "is_default_shared_subscriber_idle_timeout_secs" )] pub shared_subscriber_idle_timeout_secs: u64, + /// Upper bound on concurrently living cleanup owners (open request bodies and shared + /// subscribers). Requests beyond this are rejected with a bounded error instead of + /// growing memory without limit. + #[serde(default = "default_cleanup_queue_capacity", skip_serializing_if = "is_default_cleanup_queue_capacity")] + pub cleanup_queue_capacity: usize, + #[serde( + default = "default_recent_eviction_reentry_ttl_ms", + skip_serializing_if = "is_default_recent_eviction_reentry_ttl_ms" + )] + pub recent_eviction_reentry_ttl_ms: u64, #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_strategies: Option>, } @@ -128,6 +139,8 @@ impl Default for StreamConfigDto { throttle_kbps: 0, shared_burst_buffer_mb: default_shared_burst_buffer_mb(), shared_subscriber_idle_timeout_secs: default_shared_subscriber_idle_timeout_secs(), + cleanup_queue_capacity: default_cleanup_queue_capacity(), + recent_eviction_reentry_ttl_ms: default_recent_eviction_reentry_ttl_ms(), grace_period_hold_stream: true, hls_session_ttl_secs: default_hls_session_ttl_secs(), catchup_session_ttl_secs: default_catchup_session_ttl_secs(), @@ -147,6 +160,8 @@ impl StreamConfigDto { && self.throttle_kbps == 0 && self.shared_burst_buffer_mb == default_shared_burst_buffer_mb() && self.shared_subscriber_idle_timeout_secs == default_shared_subscriber_idle_timeout_secs() + && self.cleanup_queue_capacity == default_cleanup_queue_capacity() + && self.recent_eviction_reentry_ttl_ms == default_recent_eviction_reentry_ttl_ms() && self.grace_period_hold_stream && self.hls_session_ttl_secs == default_hls_session_ttl_secs() && self.catchup_session_ttl_secs == default_catchup_session_ttl_secs() @@ -187,6 +202,10 @@ impl StreamConfigDto { )); } + if self.cleanup_queue_capacity == 0 { + return Err(TuliproxError::ConfigStream("`cleanup_queue_capacity` must be at least 1".to_string())); + } + if let Some(strategies) = &self.admission_strategies { if let Err(err) = validate_admission_strategies(strategies, self.grace_period_millis) { return Err(TuliproxError::ConfigStream(err)); diff --git a/shared/src/model/status_check.rs b/shared/src/model/status_check.rs index 18b481223..cbef4aac7 100644 --- a/shared/src/model/status_check.rs +++ b/shared/src/model/status_check.rs @@ -18,6 +18,10 @@ pub struct StatusCheck { pub active_user_streams: Vec, #[serde(default)] pub active_provider_connections: Option, usize>>, + /// Cumulative retries quietly suppressed by the recent-eviction reentry guard. + /// A diagnostic counter, not a connection-failure metric. + #[serde(default)] + pub reentry_suppressed_total: u64, } impl Default for StatusCheck { @@ -33,6 +37,7 @@ impl Default for StatusCheck { active_user_connections: 0, active_provider_connections: None, active_user_streams: Vec::new(), + reentry_suppressed_total: 0, } } } diff --git a/shared/src/model/target_bouquet.rs b/shared/src/model/target_bouquet.rs index 5746e1f10..006defbfe 100644 --- a/shared/src/model/target_bouquet.rs +++ b/shared/src/model/target_bouquet.rs @@ -118,7 +118,7 @@ mod tests { "news".to_string(), "News".to_string(), " Zebra".to_string(), - "".to_string(), + String::new(), ]), vod: Some(vec![]), series: None, diff --git a/shared/src/utils/constants.rs b/shared/src/utils/constants.rs index 681b29d69..b93467f3b 100644 --- a/shared/src/utils/constants.rs +++ b/shared/src/utils/constants.rs @@ -38,12 +38,7 @@ const SUPPORTED_RESPONSE_HEADERS: &[&str] = &[ pub fn filter_response_header(key: &str) -> bool { SUPPORTED_RESPONSE_HEADERS.contains(&key) } -pub fn filter_request_header(key: &str) -> bool { - if key == "host" || key == "connection" { - return false; - } - true -} +pub fn filter_request_header(key: &str) -> bool { key != "host" && key != "connection" } /// Configuration for media export naming styles (Kodi, Plex, Emby, Jellyfin) pub struct ExportStyleConfig { diff --git a/shared/src/utils/flags.rs b/shared/src/utils/flags.rs index 5c83edbf8..13859e170 100644 --- a/shared/src/utils/flags.rs +++ b/shared/src/utils/flags.rs @@ -1,5 +1,5 @@ /////////// -// Create flags file with: > cargo run --bin flags_builder +// Create flags file with: > cargo run --bin flags-builder // // Case-insensitive // ================ diff --git a/test/fixtures/testkit/custom-video-fixtures.json b/test/fixtures/testkit/custom-video-fixtures.json new file mode 100644 index 000000000..6098ae2bd --- /dev/null +++ b/test/fixtures/testkit/custom-video-fixtures.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "fixtures": { + "hls/channel_unavailable.ts": "afefcdbcf3749e4e7a89321239dfcb7d2c0bdc4d2d2fb1cf4c883962c837c92f", + "hls/hls_session_or_lease_expired.ts": "9fe61499c2080f4aa1e362dfb72ec7c0a4b283f08d7dcac1f21590bd4db2fe55", + "hls/low_priority_preempted.ts": "9cc3a7b63f02e8e0bc45fe3b59c18f52443ded43fa0c9121fb0ea676def412a9", + "hls/panel_api_provisioning.ts": "b481b7b85f79883b41f8a1436301257ee5cb5a45e30993233ec73a2a5824ff13", + "hls/provider_connections_exhausted.ts": "099181ddcf9e7edc5007db6d801348afb132e27382eb4e0d33589dbe07cee790", + "hls/user_account_expired.ts": "110ea30bdec0f4d6457dcbfd2c8fb8dc0b2f517d8dedb473de1191d6de369464", + "hls/user_connections_exhausted.ts": "6ab07858d03e19b68eea8cbaecb67ad4d276633c8b423253b7f39c585cab73fe" + } +} diff --git a/test/fixtures/testkit/scenarios/evict-same-ip-latest.yml b/test/fixtures/testkit/scenarios/evict-same-ip-latest.yml new file mode 100644 index 000000000..cd542b4a0 --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-same-ip-latest.yml @@ -0,0 +1,62 @@ +schema_version: 1 +name: evict-same-ip-latest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: evict-same-ip-latest +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-b + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-c + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-c + client_ip: { mode: forwarded, value: 10.20.0.10 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-a + start: + actor: tv-a + playback_id: playback-a + session_group: playback-a + channel: live-17 + await: + valid_frames: 5 + - command_id: start-b + start: + actor: tv-b + playback_id: playback-b + session_group: playback-b + channel: live-18 + await: + valid_frames: 5 + - command_id: start-c + expect: streaming + start: + actor: tv-c + playback_id: playback-c + session_group: playback-c + channel: live-19 +policy_contract: + user_access_control: true + users: + admission-user: + max_connections: 2 + admission_strategies: + - evict_user_same_ip_latest diff --git a/test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml b/test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml new file mode 100644 index 000000000..6131b4494 --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-same-ip-oldest.yml @@ -0,0 +1,26 @@ +schema_version: 1 +name: evict-same-ip-oldest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: evict-same-ip-oldest } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-b, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/oldest-c, client_ip: { mode: forwarded, value: 10.20.0.10 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } } + - { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } } + - { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 2 } } + admission_strategies: [evict_user_same_ip_oldest] diff --git a/test/fixtures/testkit/scenarios/evict-user-latest.yml b/test/fixtures/testkit/scenarios/evict-user-latest.yml new file mode 100644 index 000000000..a5569c2b7 --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-user-latest.yml @@ -0,0 +1,26 @@ +schema_version: 1 +name: evict-user-latest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: evict-user-latest } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-b, client_ip: { mode: forwarded, value: 10.20.0.11 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-latest-c, client_ip: { mode: forwarded, value: 10.20.0.12 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } } + - { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } } + - { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 2 } } + admission_strategies: [evict_user_latest] diff --git a/test/fixtures/testkit/scenarios/evict-user-oldest.yml b/test/fixtures/testkit/scenarios/evict-user-oldest.yml new file mode 100644 index 000000000..3915384df --- /dev/null +++ b/test/fixtures/testkit/scenarios/evict-user-oldest.yml @@ -0,0 +1,26 @@ +schema_version: 1 +name: evict-user-oldest +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: evict-user-oldest } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-b, client_ip: { mode: forwarded, value: 10.20.0.11 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/user-oldest-c, client_ip: { mode: forwarded, value: 10.20.0.12 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } } + - { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } } + - { command_id: start-c, start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 }, await: { valid_frames: 5 } } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 2 } } + admission_strategies: [evict_user_oldest] diff --git a/test/fixtures/testkit/scenarios/hard-limit.yml b/test/fixtures/testkit/scenarios/hard-limit.yml new file mode 100644 index 000000000..4c7e98b14 --- /dev/null +++ b/test/fixtures/testkit/scenarios/hard-limit.yml @@ -0,0 +1,47 @@ +schema_version: 1 +name: hard-limit-denies-second-playback +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: hard-limit-denies-second-playback +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/tv-b + client_ip: { mode: forwarded, value: 10.20.0.10 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } +steps: + - command_id: start-a + start: + actor: tv-a + playback_id: playback-a + session_group: playback-a + channel: live-17 + await: + valid_frames: 5 + - command_id: start-b + expect: rejected + start: + actor: tv-b + playback_id: playback-b + session_group: playback-b + channel: live-17 +policy_contract: + user_access_control: true + users: + admission-user: + max_connections: 1 + soft_connections: 0 + admission_strategies: [] diff --git a/test/fixtures/testkit/scenarios/live-ts-same-channel-retry-latest-wins.yml b/test/fixtures/testkit/scenarios/live-ts-same-channel-retry-latest-wins.yml new file mode 100644 index 000000000..7b18f7485 --- /dev/null +++ b/test/fixtures/testkit/scenarios/live-ts-same-channel-retry-latest-wins.yml @@ -0,0 +1,58 @@ +schema_version: 1 +name: live-ts-same-channel-retry-latest-wins +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + playback_endpoint: xtream + fixture_stream: + share_live_hls: true + share_live_mpeg_ts: false + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: live-ts-same-channel-retry-latest-wins + account_limit: 1 + limit_mode: reject_new +actors: + - id: tivimate + agent: local + username: admission-user + user_agent: TiviMate/5.1.0-testkit + client_ip: { mode: forwarded, value: 10.20.0.10 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } +steps: + - command_id: start-a + start: { actor: tivimate, playback_id: playback-a, session_group: playback-a, channel: live-17 } + await: { valid_frames: 1 } + - command_id: start-b + start: { actor: tivimate, playback_id: playback-b, session_group: playback-b, channel: live-17 } + await: { valid_frames: 1 } + - command_id: start-c + start: { actor: tivimate, playback_id: playback-c, session_group: playback-c, channel: live-17 } + await: { valid_frames: 5 } + assert_origin: + active_body: 1 + max_active_body: 1 + no_limit_rejections: true + - command_id: stop-c + stop: { playback_id: playback-c } + assert_origin: + active_body: 0 + max_active_body: 1 + no_limit_rejections: true +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 1, soft_connections: 0 } + admission_strategies: + - evict_user_same_ip_latest + - evict_user_same_ip_oldest + - evict_user_latest + - evict_user_oldest + - grace_hold_stream + grace: { mode: hold_stream, timeout_millis: 2000 } + provider_max_connections: 1 + expected_provider_slots: 1 diff --git a/test/fixtures/testkit/scenarios/provider-hard-limit.yml b/test/fixtures/testkit/scenarios/provider-hard-limit.yml new file mode 100644 index 000000000..688bbe923 --- /dev/null +++ b/test/fixtures/testkit/scenarios/provider-hard-limit.yml @@ -0,0 +1,38 @@ +schema_version: 1 +name: provider-hard-limit-denies-distinct-stream +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: provider-hard-limit-denies-distinct-stream +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/provider-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/provider-b + client_ip: { mode: forwarded, value: 10.20.0.11 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } +steps: + - command_id: start-a + start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 } + await: { valid_frames: 5 } + - command_id: start-b + expect: rejected + start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 } +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 2 } + admission_strategies: [] + provider_max_connections: 1 diff --git a/test/fixtures/testkit/scenarios/reentry-suppresses-evicted-retry.yml b/test/fixtures/testkit/scenarios/reentry-suppresses-evicted-retry.yml new file mode 100644 index 000000000..0f2365704 --- /dev/null +++ b/test/fixtures/testkit/scenarios/reentry-suppresses-evicted-retry.yml @@ -0,0 +1,30 @@ +schema_version: 1 +name: reentry-suppresses-evicted-retry +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: reentry-suppresses-evicted-retry } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/reentry-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/reentry-b, client_ip: { mode: forwarded, value: 10.20.0.10 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } +steps: + - { command_id: start-a, start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 }, await: { valid_frames: 5 } } + - { command_id: start-b, start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 }, await: { valid_frames: 5 } } + # tv-a retries the channel it was just evicted from. The reentry guard must + # suppress it quietly (HTTP 204) and keep playback-b streaming instead of + # evicting playback-b back (playback ping-pong). + - command_id: retry-a + expect: suppressed + start: { actor: tv-a, playback_id: playback-a-retry, session_group: playback-a-retry, channel: live-17 } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 1 } } + admission_strategies: [evict_user_oldest] + recent_eviction_reentry_ttl_ms: 10000 diff --git a/test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml b/test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml new file mode 100644 index 000000000..86e09b23d --- /dev/null +++ b/test/fixtures/testkit/scenarios/shared-stream-single-provider-slot.yml @@ -0,0 +1,38 @@ +schema_version: 1 +name: shared-stream-uses-one-provider-slot +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: shared-stream-uses-one-provider-slot +actors: + - id: tv-a + agent: local + username: admission-user + user_agent: tuliprox-testkit/shared-a + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: tv-b + agent: local + username: admission-user + user_agent: tuliprox-testkit/shared-b + client_ip: { mode: forwarded, value: 10.20.0.11 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } +steps: + - command_id: start-a + start: { actor: tv-a, playback_id: playback-a, session_group: shared-live-17, channel: live-17 } + await: { valid_frames: 5 } + - command_id: start-b + start: { actor: tv-b, playback_id: playback-b, session_group: shared-live-17, channel: live-17 } + await: { valid_frames: 5 } +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 2 } + admission_strategies: [] + provider_max_connections: 1 + expected_provider_slots: 1 diff --git a/test/fixtures/testkit/scenarios/soft-slot-exhausts-without-upstream-leak.yml b/test/fixtures/testkit/scenarios/soft-slot-exhausts-without-upstream-leak.yml new file mode 100644 index 000000000..394cc5c65 --- /dev/null +++ b/test/fixtures/testkit/scenarios/soft-slot-exhausts-without-upstream-leak.yml @@ -0,0 +1,50 @@ +schema_version: 1 +name: soft-slot-exhausts-without-upstream-leak +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: soft-slot-exhausts-without-upstream-leak + account_limit: 2 + limit_mode: reject_new +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/exhaust-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/exhaust-b, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/exhaust-c, client_ip: { mode: forwarded, value: 10.20.0.10 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-a + start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 } + await: { valid_frames: 5 } + assert_origin: { active_body: 1, no_limit_rejections: true } + assert_runtime: { active_users: 1, active_user_connections: 1 } + - command_id: start-b + start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 } + await: { valid_frames: 5 } + assert_origin: { active_body: 2, no_limit_rejections: true } + assert_runtime: { active_users: 1, active_user_connections: 2 } + - command_id: start-c + expect: rejected + start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 } + assert_origin: { active_body: 2, no_limit_rejections: true } + assert_runtime: { active_users: 1, active_user_connections: 2 } + - command_id: stop-a + stop: { playback_id: playback-a } + assert_origin: { active_body: 1 } + assert_runtime: { active_users: 1, active_user_connections: 1 } + - command_id: stop-b + stop: { playback_id: playback-b } + assert_origin: { active_body: 0 } + assert_runtime: { active_users: 0, active_user_connections: 0 } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 1, soft_connections: 1 } } + admission_strategies: [] + provider_max_connections: 2 diff --git a/test/fixtures/testkit/scenarios/soft-slot-precedes-eviction.yml b/test/fixtures/testkit/scenarios/soft-slot-precedes-eviction.yml new file mode 100644 index 000000000..ea8a85d22 --- /dev/null +++ b/test/fixtures/testkit/scenarios/soft-slot-precedes-eviction.yml @@ -0,0 +1,41 @@ +schema_version: 1 +name: soft-slot-precedes-eviction +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: { run_id: soft-slot-precedes-eviction } +actors: + - { id: tv-a, agent: local, username: admission-user, user_agent: tuliprox-testkit/soft-a, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-b, agent: local, username: admission-user, user_agent: tuliprox-testkit/soft-b, client_ip: { mode: forwarded, value: 10.20.0.10 } } + - { id: tv-c, agent: local, username: admission-user, user_agent: tuliprox-testkit/soft-c, client_ip: { mode: forwarded, value: 10.20.0.10 } } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-a + start: { actor: tv-a, playback_id: playback-a, session_group: playback-a, channel: live-17 } + await: { valid_frames: 5 } + assert_runtime: { active_users: 1, active_user_connections: 1 } + - command_id: start-b + start: { actor: tv-b, playback_id: playback-b, session_group: playback-b, channel: live-18 } + await: { valid_frames: 5 } + assert_runtime: { active_users: 1, active_user_connections: 2 } + - command_id: start-c + start: { actor: tv-c, playback_id: playback-c, session_group: playback-c, channel: live-19 } + await: { valid_frames: 5 } + assert_runtime: { active_users: 1, active_user_connections: 2 } + - command_id: stop-a + stop: { playback_id: playback-a } + assert_runtime: { active_users: 1, active_user_connections: 1 } + - command_id: stop-c + stop: { playback_id: playback-c } + assert_runtime: { active_users: 0, active_user_connections: 0 } +policy_contract: + user_access_control: true + users: { admission-user: { max_connections: 1, soft_connections: 1 } } + admission_strategies: [evict_user_latest] diff --git a/test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml b/test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml new file mode 100644 index 000000000..fa2824b90 --- /dev/null +++ b/test/fixtures/testkit/scenarios/vod-range-reopen-preserves-three-live.yml @@ -0,0 +1,99 @@ +schema_version: 1 +name: vod-range-reopen-preserves-three-live +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: vod-range-reopen-preserves-three-live + account_limit: 5 + limit_mode: reject_new +actors: + - id: live-viewer-1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-1 + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: live-viewer-2 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-2 + client_ip: { mode: forwarded, value: 10.20.0.11 } + - id: live-viewer-3 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-3 + client_ip: { mode: forwarded, value: 10.20.0.12 } + - id: vod-viewer-ua1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/vod-player-ua1 + client_ip: { mode: forwarded, value: 10.20.0.13 } + - id: vod-viewer-ua2 + agent: local + username: admission-user + user_agent: tuliprox-testkit/vod-player-ua2 + client_ip: { mode: forwarded, value: 10.20.0.13 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-live-1 + start: { actor: live-viewer-1, playback_id: live-1, session_group: live-1, channel: live-17 } + await: { valid_frames: 3 } + - command_id: start-live-2 + start: { actor: live-viewer-2, playback_id: live-2, session_group: live-2, channel: live-18 } + await: { valid_frames: 3 } + - command_id: start-live-3 + start: { actor: live-viewer-3, playback_id: live-3, session_group: live-3, channel: live-19 } + await: { valid_frames: 3 } + assert_origin: + active_body: 3 + max_active_body: 3 + max_active_tcp: 3 + - command_id: start-vod-probe-1 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-1 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=0-524287" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + - command_id: start-vod-probe-2 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-2 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=1048576-1572863" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + no_limit_rejections: true + no_evictions: true + - command_id: stop-vod-probe-2 + stop: { playback_id: vod-probe-2 } + - command_id: stop-vod-probe-1 + stop: { playback_id: vod-probe-1 } + assert_origin: + active_body: 3 + max_active_body: 4 + max_active_tcp: 4 +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 10 } + admission_strategies: [] + provider_max_connections: 4 diff --git a/test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml b/test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml new file mode 100644 index 000000000..53167db15 --- /dev/null +++ b/test/fixtures/testkit/scenarios/vod-range-reopen-strict-cap.yml @@ -0,0 +1,94 @@ +schema_version: 1 +name: vod-range-reopen-strict-cap +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: vod-range-reopen-strict-cap + account_limit: 4 + limit_mode: reject_new +actors: + - id: live-viewer-1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-1 + client_ip: { mode: forwarded, value: 10.20.0.10 } + - id: live-viewer-2 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-2 + client_ip: { mode: forwarded, value: 10.20.0.11 } + - id: live-viewer-3 + agent: local + username: admission-user + user_agent: tuliprox-testkit/live-3 + client_ip: { mode: forwarded, value: 10.20.0.12 } + - id: vod-viewer-ua1 + agent: local + username: admission-user + user_agent: tuliprox-testkit/vod-player-ua1 + client_ip: { mode: forwarded, value: 10.20.0.13 } +channels: + live-17: { origin_marker: 17, protocol: xtream_ts } + live-18: { origin_marker: 18, protocol: xtream_ts } + live-19: { origin_marker: 19, protocol: xtream_ts } +steps: + - command_id: start-live-1 + start: { actor: live-viewer-1, playback_id: live-1, session_group: live-1, channel: live-17 } + await: { valid_frames: 3 } + - command_id: start-live-2 + start: { actor: live-viewer-2, playback_id: live-2, session_group: live-2, channel: live-18 } + await: { valid_frames: 3 } + - command_id: start-live-3 + start: { actor: live-viewer-3, playback_id: live-3, session_group: live-3, channel: live-19 } + await: { valid_frames: 3 } + assert_origin: + active_body: 3 + max_active_body: 3 + max_active_tcp: 3 + - command_id: start-vod-probe-1 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-1 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=0-524287" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + - command_id: start-vod-probe-2 + start: + actor: vod-viewer-ua1 + playback_id: vod-probe-2 + session_group: vod-movie + vod_object: movie.mkv + range: "bytes=1048576-1572863" + read_limit_bytes: 32768 + post_read_action: pause + assert_origin: + active_body: 4 + max_active_body: 4 + max_active_tcp: 4 + no_limit_rejections: true + no_evictions: true + - command_id: stop-vod-probe-2 + stop: { playback_id: vod-probe-2 } + - command_id: stop-vod-probe-1 + stop: { playback_id: vod-probe-1 } + assert_origin: + active_body: 3 + max_active_body: 4 + max_active_tcp: 4 +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 10 } + admission_strategies: [] + provider_max_connections: 4 diff --git a/test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml b/test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml new file mode 100644 index 000000000..aa73f4843 --- /dev/null +++ b/test/fixtures/testkit/scenarios/vod-reopen-backpressured-body.yml @@ -0,0 +1,65 @@ +schema_version: 1 +name: vod-reopen-backpressured-body +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: vod-reopen-backpressured-body + account_limit: 2 + limit_mode: reject_new +actors: + - id: vod-viewer + agent: local + username: admission-user + user_agent: tuliprox-testkit/backpressured-vod + client_ip: { mode: forwarded, value: 10.20.0.20 } +channels: {} +steps: + # First probe: read only 16 KiB out of 512 KiB, then pause to create downstream backpressure + - command_id: start-vod-backpressured + start: + actor: vod-viewer + playback_id: vod-bp-1 + session_group: vod-bp + vod_object: movie.mkv + range: "bytes=0-524287" + read_limit_bytes: 16384 + post_read_action: pause + assert_origin: + active_body: 1 + max_active_body: 1 + max_active_tcp: 1 + # Second probe on the same session while first probe is still open and backpressured. + # Forced reopen must reclaim the single provider slot (max_connections: 1) without peak violations. + - command_id: start-vod-next + start: + actor: vod-viewer + playback_id: vod-bp-2 + session_group: vod-bp + vod_object: movie.mkv + range: "bytes=1048576-1572863" + read_limit_bytes: 65536 + post_read_action: pause + assert_origin: + active_body: 1 + max_active_body: 1 + max_active_tcp: 1 + no_limit_rejections: true + - command_id: stop-vod-next + stop: { playback_id: vod-bp-2 } + - command_id: stop-vod-backpressured + stop: { playback_id: vod-bp-1 } + assert_origin: + active_body: 0 + max_active_body: 1 + max_active_tcp: 1 +policy_contract: + user_access_control: true + users: + admission-user: { max_connections: 2 } + admission_strategies: [] + provider_max_connections: 1 diff --git a/tools/tuliprox-testkit/Cargo.toml b/tools/tuliprox-testkit/Cargo.toml new file mode 100644 index 000000000..8bfae1960 --- /dev/null +++ b/tools/tuliprox-testkit/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "tuliprox-testkit" +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +axum = { version = "0.8.9", features = ["macros", "default", "ws"] } +base64.workspace = true +blake3.workspace = true +bytes.workspace = true +clap = { version = "4.6.6", features = ["derive"] } +crc32fast = "1.5.1" +futures.workspace = true +reqwest = { version = "0.13.4", features = ["json", "stream", "rustls"] } +serde.workspace = true +serde_json.workspace = true +serde-saphyr.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "process", "signal", "sync", "time"] } +tokio-tungstenite = "0.29.0" +tempfile = "3.27.0" +pin-project-lite = "0.2.17" +url.workspace = true +hyper = { version = "1.11.0", features = ["server", "http1"] } +hyper-util = { version = "0.1.20", features = ["server", "server-auto", "tokio"] } +tower-service = "0.3.3" + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } +tokio-stream = "0.1.19" diff --git a/tools/tuliprox-testkit/src/adaptive.rs b/tools/tuliprox-testkit/src/adaptive.rs new file mode 100644 index 000000000..7a94523ad --- /dev/null +++ b/tools/tuliprox-testkit/src/adaptive.rs @@ -0,0 +1,103 @@ +use crate::TestkitError; +use std::{collections::HashMap, time::Duration}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdaptiveSessionState { + Prepared, + Active, + Preserved, +} + +#[derive(Debug)] +struct Session { + state: AdaptiveSessionState, + expires_at: tokio::time::Instant, +} + +#[derive(Debug, Default)] +pub struct AdaptiveSessionTracker { + sessions: HashMap, +} + +impl AdaptiveSessionTracker { + pub fn manifest(&mut self, session_group: String, ttl: Duration) -> Result { + self.reap_expired(); + if self.sessions.contains_key(&session_group) { + return Err(TestkitError::Protocol("adaptive manifest created a duplicate logical session".to_owned())); + } + self.sessions.insert( + session_group, + Session { state: AdaptiveSessionState::Prepared, expires_at: tokio::time::Instant::now() + ttl }, + ); + Ok(AdaptiveSessionState::Prepared) + } + + pub fn segment(&mut self, session_group: &str, ttl: Duration) -> Result { + let session = self.sessions.get_mut(session_group).ok_or_else(|| { + TestkitError::Configuration(format!("adaptive segment has no prepared session {session_group}")) + })?; + if tokio::time::Instant::now() > session.expires_at { + return Err(TestkitError::Protocol("adaptive session TTL expired before segment activation".to_owned())); + } + session.state = AdaptiveSessionState::Active; + session.expires_at = tokio::time::Instant::now() + ttl; + Ok(session.state) + } + + pub fn preserve(&mut self, session_group: &str, ttl: Duration) -> Result<(), TestkitError> { + self.reap_expired(); + let session = self.sessions.get_mut(session_group).ok_or_else(|| { + TestkitError::Configuration(format!("cannot preserve unknown adaptive session {session_group}")) + })?; + session.state = AdaptiveSessionState::Preserved; + session.expires_at = tokio::time::Instant::now() + ttl; + Ok(()) + } + + pub fn reap_expired(&mut self) { + self.sessions.retain(|_, session| tokio::time::Instant::now() <= session.expires_at); + } + + #[must_use] + pub fn state(&self, session_group: &str) -> Option { + self.sessions.get(session_group).map(|session| session.state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn segment_activates_and_reacquisition_preserves_one_session() { + let mut tracker = AdaptiveSessionTracker::default(); + assert_eq!( + tracker.manifest("hls-a".to_owned(), Duration::from_secs(10)).unwrap(), + AdaptiveSessionState::Prepared + ); + assert_eq!(tracker.segment("hls-a", Duration::from_secs(10)).unwrap(), AdaptiveSessionState::Active); + tracker.preserve("hls-a", Duration::from_secs(10)).unwrap(); + assert_eq!(tracker.segment("hls-a", Duration::from_secs(10)).unwrap(), AdaptiveSessionState::Active); + } + + #[tokio::test(start_paused = true)] + async fn manifest_reaps_expired_session_before_duplicate_check() { + let mut tracker = AdaptiveSessionTracker::default(); + tracker.manifest("hls-a".to_owned(), Duration::from_secs(5)).unwrap(); + tokio::time::advance(Duration::from_secs(6)).await; + + assert_eq!( + tracker.manifest("hls-a".to_owned(), Duration::from_secs(10)).unwrap(), + AdaptiveSessionState::Prepared + ); + } + + #[tokio::test(start_paused = true)] + async fn preserve_rejects_expired_unknown_session() { + let mut tracker = AdaptiveSessionTracker::default(); + tracker.manifest("hls-a".to_owned(), Duration::from_secs(5)).unwrap(); + tokio::time::advance(Duration::from_secs(6)).await; + + assert!(matches!(tracker.preserve("hls-a", Duration::from_secs(10)), Err(TestkitError::Configuration(_)))); + } +} diff --git a/tools/tuliprox-testkit/src/bootstrap.rs b/tools/tuliprox-testkit/src/bootstrap.rs new file mode 100644 index 000000000..4f5fc5dee --- /dev/null +++ b/tools/tuliprox-testkit/src/bootstrap.rs @@ -0,0 +1,534 @@ +use crate::{ + config::{BootstrapConfig, Channel, PolicyContract}, + TestkitError, +}; +use std::{ + collections::HashMap, + fmt::Write, + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; +use tempfile::TempDir; +use tokio::process::{Child, Command}; + +const FIXTURE_PASSWORD: &str = "testkit-password"; + +/// Files owned by one isolated run. They are written below a unique temporary +/// directory and must never refer to an operator's configuration or storage. +pub struct FixturePaths { + _directory: TempDir, + pub root: PathBuf, + pub config_file: PathBuf, + pub source_file: PathBuf, + pub api_proxy_file: PathBuf, + pub web_root: PathBuf, +} + +impl FixturePaths { + pub fn create( + run_id: &str, + origin_address: SocketAddr, + tuliprox_address: SocketAddr, + policy: Option<&PolicyContract>, + channels: &HashMap, + fixture_stream: &crate::config::FixtureStreamOptions, + add_xtream_output: bool, + ) -> Result { + let directory = tempfile::Builder::new().prefix("tuliprox-testkit-").tempdir()?; + let root = directory.path().to_path_buf(); + let web_root = root.join("web"); + let storage_dir = root.join("storage"); + let backup_dir = root.join("backup"); + let history_dir = root.join("history"); + std::fs::create_dir_all(web_root.join("static/.well-known"))?; + std::fs::create_dir_all(&storage_dir)?; + std::fs::create_dir_all(&backup_dir)?; + std::fs::create_dir_all(&history_dir)?; + + let config_file = root.join("config.yml"); + let source_file = root.join("source.yml"); + let api_proxy_file = root.join("api-proxy.yml"); + std::fs::write( + &config_file, + render_config(tuliprox_address, &web_root, &storage_dir, &backup_dir, &history_dir, policy)?, + )?; + std::fs::write( + &source_file, + render_sources(run_id, origin_address, channels, policy, fixture_stream, add_xtream_output), + )?; + std::fs::write(&api_proxy_file, render_api_proxy(tuliprox_address, policy))?; + + Ok(Self { _directory: directory, root, config_file, source_file, api_proxy_file, web_root }) + } +} + +#[must_use] +pub const fn fixture_password() -> &'static str { FIXTURE_PASSWORD } + +fn render_config( + tuliprox_address: SocketAddr, + web_root: &Path, + storage_dir: &Path, + backup_dir: &Path, + history_dir: &Path, + policy: Option<&PolicyContract>, +) -> Result { + let user_access_control = policy.is_some_and(|contract| contract.user_access_control); + let grace = policy.and_then(|contract| contract.grace.as_ref()); + let grace_millis = grace.map_or(0, |contract| contract.timeout_millis); + let grace_hold = grace.is_none_or(|contract| matches!(contract.mode, crate::oracle::GraceMode::HoldStream)); + let mut config = format!( + "api:\n host: 127.0.0.1\n port: {}\n web_root: {}\nworking_dir: {}\nbackup_dir: {}\nuser_access_control: {user_access_control}\nupdate_on_boot: true\nlog:\n log_level: debug\nweb_ui:\n enabled: true\n user_ui_enabled: false\nreverse_proxy:\n stream:\n grace_period_millis: {grace_millis}\n grace_period_hold_stream: {grace_hold}\n shared_subscriber_idle_timeout_secs: 1\n", + tuliprox_address.port(), + yaml_scalar(web_root), + yaml_scalar(storage_dir), + yaml_scalar(backup_dir), + ); + if let Some(policy) = policy { + if let Some(strategies) = &policy.admission_strategies { + if strategies.is_empty() { + config.push_str(" admission_strategies: []\n"); + } else { + config.push_str(" admission_strategies:\n"); + for strategy in strategies { + let encoded = serde_json::to_string(strategy).map_err(|error| { + TestkitError::Configuration(format!("cannot render admission strategy: {error}")) + })?; + let _ = writeln!(config, " - {}", encoded.trim_matches('\"')); + } + } + } + if let Some(ttl_ms) = policy.recent_eviction_reentry_ttl_ms { + let _ = writeln!(config, " recent_eviction_reentry_ttl_ms: {ttl_ms}"); + } + } + let _ = writeln!( + config, + " stream_history:\n stream_history_enabled: true\n stream_history_directory: {}\n rewrite_secret: 0123456789abcdef0123456789abcdef", + yaml_scalar(history_dir) + ); + Ok(config) +} + +fn render_sources( + run_id: &str, + origin_address: SocketAddr, + channels: &HashMap, + policy: Option<&PolicyContract>, + fixture_stream: &crate::config::FixtureStreamOptions, + add_xtream_output: bool, +) -> String { + let mut markers = channels.values().map(|channel| channel.origin_marker).collect::>(); + markers.sort_unstable(); + markers.dedup(); + if markers.is_empty() { + markers.push(17); + } + let marker_comment = markers.iter().map(u32::to_string).collect::>().join(","); + let provider_limit = policy + .and_then(|contract| contract.provider_max_connections) + .map_or_else(String::new, |limit| format!(" max_connections: {limit}\n")); + let share_hls = fixture_stream.share_live_hls; + let share_mpeg_ts = fixture_stream.share_live_mpeg_ts; + let extra_output = if add_xtream_output { " - type: xtream\n" } else { "" }; + format!( + "inputs:\n - name: testkit-origin\n type: m3u\n url: http://{origin_address}/catalog/input.m3u?run={run_id}\n{provider_limit}sources:\n - inputs:\n - testkit-origin\n targets:\n - name: testkit\n output:\n - type: m3u\n{extra_output} options:\n share_live_streams:\n hls: {share_hls}\n mpeg_ts: {share_mpeg_ts}\n# Origin markers for this fixture: {marker_comment}\n" + ) +} + +fn render_api_proxy(tuliprox_address: SocketAddr, policy: Option<&PolicyContract>) -> String { + let mut output = format!( + "server:\n - name: testkit\n protocol: http\n host: 127.0.0.1\n port: \"{}\"\n timezone: UTC\n message: Tuliprox testkit\nuser:\n - target: testkit\n credentials:\n", + tuliprox_address.port() + ); + if let Some(policy) = policy { + for (username, user) in &policy.users { + let _ = writeln!( + output, + " - username: {}\n password: {}\n proxy: reverse\n server: testkit\n max_connections: {}\n soft_connections: {}\n status: Active", + yaml_string(username), + FIXTURE_PASSWORD, + user.max_connections, + user.soft_connections, + ); + } + } + if !output.contains(" - username:") { + output.push_str(" - username: testkit\n password: testkit-password\n proxy: reverse\n server: testkit\n status: Active\n"); + } + output +} + +fn yaml_scalar(path: &Path) -> String { yaml_string(&path.to_string_lossy()) } + +fn yaml_string(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } + +/// Owns only the SUT process launched for this run and waits for its explicit readiness endpoint. +pub struct IsolatedSut { + child: Child, +} + +/// Owns every process and every generated file used by one self-contained +/// Tuliprox scenario. The fixture never adopts an externally running process. +pub struct IsolatedFixture { + pub paths: FixturePaths, + pub base_url: String, + pub origin_control_url: String, + origin: Child, + sut: IsolatedSut, +} + +impl IsolatedFixture { + #[allow(clippy::too_many_arguments)] + pub async fn start( + bootstrap: &BootstrapConfig, + working_directory: &Path, + testkit_binary: &Path, + run_id: &str, + origin_config: Option<&crate::config::OriginConfig>, + policy: Option<&PolicyContract>, + channels: &HashMap, + fixture_stream: &crate::config::FixtureStreamOptions, + add_xtream_output: bool, + ) -> Result { + const MAX_START_ATTEMPTS: usize = 5; + let mut last_error = None; + + for attempt in 1..=MAX_START_ATTEMPTS { + let attempt_run_id = if attempt == 1 { run_id.to_string() } else { format!("{run_id}-retry{attempt}") }; + match Self::start_attempt( + bootstrap, + working_directory, + testkit_binary, + &attempt_run_id, + origin_config, + policy, + channels, + fixture_stream, + add_xtream_output, + ) + .await + { + Ok(env) => { + if attempt > 1 { + eprintln!("Bootstrap succeeded on attempt {attempt}/{MAX_START_ATTEMPTS}"); + } + return Ok(env); + } + Err(error) => { + eprintln!("Bootstrap start attempt {attempt}/{MAX_START_ATTEMPTS} failed: {error}; retrying with fresh ports..."); + last_error = Some(error); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + + Err(last_error.unwrap_or_else(|| TestkitError::Protocol("failed to bootstrap environment".to_owned()))) + } + + #[allow(clippy::too_many_arguments)] + async fn start_attempt( + bootstrap: &BootstrapConfig, + working_directory: &Path, + testkit_binary: &Path, + run_id: &str, + origin_config: Option<&crate::config::OriginConfig>, + policy: Option<&PolicyContract>, + channels: &HashMap, + fixture_stream: &crate::config::FixtureStreamOptions, + add_xtream_output: bool, + ) -> Result { + let origin_address = reserve_loopback_address()?; + let origin_control_address = reserve_loopback_address()?; + let tuliprox_address = reserve_loopback_address()?; + let paths = FixturePaths::create( + run_id, + origin_address, + tuliprox_address, + policy, + channels, + fixture_stream, + add_xtream_output, + )?; + let markers = fixture_markers(channels); + let mut origin_cmd = Command::new(testkit_binary); + origin_cmd + .arg("origin") + .arg("--listen") + .arg(origin_address.to_string()) + .arg("--control-listen") + .arg(origin_control_address.to_string()) + .arg("--run-id") + .arg(run_id) + .arg("--markers") + .arg(markers); + if let Some(origin_cfg) = origin_config { + if let Some(limit) = origin_cfg.account_limit { + origin_cmd.arg("--account-limit").arg(limit.to_string()); + } + if let Some(mode) = origin_cfg.limit_mode { + let mode_str = match mode { + crate::origin_transport::OriginLimitMode::ObserveOnly => "observe_only", + crate::origin_transport::OriginLimitMode::RejectNew => "reject_new", + crate::origin_transport::OriginLimitMode::EvictOldest => "evict_oldest", + }; + origin_cmd.arg("--limit-mode").arg(mode_str); + } + } + let mut origin = origin_cmd.current_dir(working_directory).spawn()?; + let origin_health = format!("http://{origin_control_address}/health"); + if let Err(error) = wait_for_http_ready(&mut origin, &origin_health, Duration::from_secs(10)).await { + let _ = stop_child(&mut origin).await; + return Err(error); + } + let base_url = format!("http://{tuliprox_address}"); + let rendered = render_bootstrap(bootstrap, &paths, &base_url)?; + let sut = match IsolatedSut::start(&rendered, working_directory).await { + Ok(sut) => sut, + Err(error) => { + let _ = stop_child(&mut origin).await; + return Err(error); + } + }; + let origin_control_url = format!("http://{origin_control_address}"); + Ok(Self { paths, base_url, origin_control_url, origin, sut }) + } + + pub async fn stop(mut self) -> Result<(), TestkitError> { + let sut_result = self.sut.stop().await; + let origin_result = stop_child(&mut self.origin).await; + match (sut_result, origin_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(sut_error), Err(origin_error)) => Err(TestkitError::Protocol(format!( + "Tuliprox cleanup failed ({sut_error}); origin cleanup failed ({origin_error})" + ))), + } + } +} + +fn reserve_loopback_address() -> Result { + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let address = listener.local_addr()?; + drop(listener); + Ok(address) +} + +fn fixture_markers(channels: &HashMap) -> String { + let mut markers = channels.values().map(|channel| channel.origin_marker).collect::>(); + markers.sort_unstable(); + markers.dedup(); + if markers.is_empty() { + markers.push(17); + } + markers.iter().map(u32::to_string).collect::>().join(",") +} + +fn render_bootstrap( + bootstrap: &BootstrapConfig, + paths: &FixturePaths, + base_url: &str, +) -> Result { + let render = |value: &str| { + value + .replace("{config_file}", &paths.config_file.to_string_lossy()) + .replace("{source_file}", &paths.source_file.to_string_lossy()) + .replace("{api_proxy_file}", &paths.api_proxy_file.to_string_lossy()) + .replace("{run_directory}", &paths.root.to_string_lossy()) + .replace("{base_url}", base_url) + }; + let command = resolve_environment_command(&render(&bootstrap.command))?; + Ok(BootstrapConfig { + command, + arguments: bootstrap.arguments.iter().map(|argument| render(argument)).collect(), + readiness_url: render(&bootstrap.readiness_url), + readiness_timeout_millis: bootstrap.readiness_timeout_millis, + }) +} + +fn resolve_environment_command(command: &str) -> Result { + let Some(variable) = command.strip_prefix("${env:").and_then(|value| value.strip_suffix('}')) else { + return Ok(command.to_owned()); + }; + std::env::var(variable).map_err(|_| { + TestkitError::Configuration(format!("bootstrap command environment variable {variable} is not set")) + }) +} + +async fn wait_for_http_ready(child: &mut Child, readiness_url: &str, timeout: Duration) -> Result<(), TestkitError> { + let deadline = tokio::time::Instant::now() + timeout; + let client = reqwest::Client::new(); + loop { + if let Some(status) = child.try_wait()? { + return Err(TestkitError::Protocol(format!("fixture process exited before readiness: {status}"))); + } + if client.get(readiness_url).send().await.is_ok_and(|response| response.status().is_success()) { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!("fixture process did not become ready at {readiness_url}"))); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn stop_child(child: &mut Child) -> Result<(), TestkitError> { + if child.try_wait()?.is_some() { + return Ok(()); + } + child.start_kill()?; + tokio::time::timeout(Duration::from_secs(5), child.wait()) + .await + .map_err(|_| TestkitError::Protocol("fixture process did not exit during cleanup".to_owned()))??; + Ok(()) +} + +impl IsolatedSut { + pub async fn start(config: &BootstrapConfig, working_directory: &Path) -> Result { + if config.command.trim().is_empty() || config.readiness_url.trim().is_empty() { + return Err(TestkitError::Configuration("bootstrap command and readiness URL are required".to_owned())); + } + let child = Command::new(&config.command).args(&config.arguments).current_dir(working_directory).spawn()?; + let mut sut = Self { child }; + if let Err(startup_error) = + sut.wait_ready(&config.readiness_url, Duration::from_millis(config.readiness_timeout_millis.max(1))).await + { + if let Err(cleanup_error) = sut.stop().await { + return Err(TestkitError::Protocol(format!( + "isolated Tuliprox startup failed ({startup_error}) and cleanup failed ({cleanup_error})" + ))); + } + return Err(startup_error); + } + Ok(sut) + } + + async fn wait_ready(&mut self, readiness_url: &str, timeout: Duration) -> Result<(), TestkitError> { + let deadline = tokio::time::Instant::now() + timeout; + let client = reqwest::Client::new(); + loop { + if let Some(status) = self.child.try_wait()? { + return Err(TestkitError::Protocol(format!( + "isolated Tuliprox process exited before readiness: {status}" + ))); + } + if let Ok(response) = client.get(readiness_url).send().await { + if response.status().is_success() { + return Ok(()); + } + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol("isolated Tuliprox process did not become ready".to_owned())); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + pub async fn stop(mut self) -> Result<(), TestkitError> { + if self.child.try_wait()?.is_none() { + self.child.start_kill()?; + self.child.wait().await?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn early_process_exit_does_not_wait_for_the_readiness_deadline() { + let config = BootstrapConfig { + command: "/bin/false".to_owned(), + arguments: Vec::new(), + readiness_url: "http://127.0.0.1:1/ready".to_owned(), + readiness_timeout_millis: 5_000, + }; + let started_at = tokio::time::Instant::now(); + let result = IsolatedSut::start(&config, Path::new("/tmp")).await; + assert!(matches!(result, Err(TestkitError::Protocol(message)) if message.contains("exited before readiness"))); + assert!(started_at.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn fixture_files_are_run_scoped_and_contain_no_operator_paths() -> Result<(), TestkitError> { + let policy = PolicyContract { + user_access_control: true, + users: HashMap::from([( + "fixture-user".to_owned(), + crate::config::UserPolicy { max_connections: 1, soft_connections: 0 }, + )]), + admission_strategies: Some(vec![crate::oracle::AdmissionStrategy::EvictUserSameIpLatest]), + recent_eviction_reentry_ttl_ms: None, + grace: None, + provider_max_connections: None, + expected_provider_slots: None, + }; + let channels = HashMap::from([( + "fixture-channel".to_owned(), + Channel { origin_marker: 17, protocol: "xtream_ts".to_owned() }, + )]); + let fixture = FixturePaths::create( + "fixture-run", + "127.0.0.1:9910".parse::().map_err(|error| TestkitError::Configuration(error.to_string()))?, + "127.0.0.1:8901".parse::().map_err(|error| TestkitError::Configuration(error.to_string()))?, + Some(&policy), + &channels, + &crate::config::FixtureStreamOptions::default(), + false, + )?; + let config = std::fs::read_to_string(&fixture.config_file)?; + let source = std::fs::read_to_string(&fixture.source_file)?; + let api_proxy = std::fs::read_to_string(&fixture.api_proxy_file)?; + + assert!(config.contains(fixture.root.to_string_lossy().as_ref())); + assert!(source.contains("fixture-run")); + assert!(api_proxy.contains("fixture-user")); + let expected_paths = [ + fixture.root.join("web"), + fixture.root.join("storage"), + fixture.root.join("backup"), + fixture.root.join("history"), + ]; + for expected in &expected_paths { + assert!( + expected.starts_with(&fixture.root), + "fixture path {} must stay under the fixture root {}", + expected.display(), + fixture.root.display() + ); + assert!( + config.contains(expected.to_string_lossy().as_ref()), + "rendered config must reference fixture path {}", + expected.display() + ); + } + Ok(()) + } + + #[test] + fn render_sources_with_mpeg_ts_sharing_disabled() { + let opts = crate::config::FixtureStreamOptions { share_live_hls: true, share_live_mpeg_ts: false }; + let source = render_sources("run", "127.0.0.1:9910".parse().unwrap(), &HashMap::new(), None, &opts, false); + assert!(source.contains("mpeg_ts: false")); + assert!(source.contains("hls: true")); + assert!(!source.contains("type: xtream")); + } + + #[test] + fn render_sources_includes_xtream_output_when_requested() { + let opts = crate::config::FixtureStreamOptions::default(); + let source = render_sources("run", "127.0.0.1:9910".parse().unwrap(), &HashMap::new(), None, &opts, true); + assert!(source.contains("type: xtream")); + } + + #[test] + fn render_sources_omits_xtream_output_by_default() { + let opts = crate::config::FixtureStreamOptions::default(); + let source = render_sources("run", "127.0.0.1:9910".parse().unwrap(), &HashMap::new(), None, &opts, false); + assert!(!source.contains("type: xtream")); + assert!(source.contains("mpeg_ts: true")); + } +} diff --git a/tools/tuliprox-testkit/src/config.rs b/tools/tuliprox-testkit/src/config.rs new file mode 100644 index 000000000..14b21331e --- /dev/null +++ b/tools/tuliprox-testkit/src/config.rs @@ -0,0 +1,753 @@ +use crate::{ + oracle::{AdmissionStrategy, GraceMode}, + TestkitError, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, HashSet}, + fs, + path::Path, + time::Duration, +}; + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scenario { + pub schema_version: u16, + pub name: String, + pub tuliprox: TuliproxConfig, + #[serde(default)] + pub origin: Option, + #[serde(default)] + pub actors: Vec, + #[serde(default)] + pub channels: HashMap, + #[serde(default)] + pub agents: AgentsConfig, + #[serde(default)] + pub policy_contract: Option, + #[serde(default)] + pub steps: Vec, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AgentsConfig { + #[serde(default)] + pub required: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyContract { + pub user_access_control: bool, + #[serde(default)] + pub users: HashMap, + /// `None` preserves an omitted field, which has distinct Tuliprox semantics + /// from an explicit empty admission-strategy list. + #[serde(default)] + pub admission_strategies: Option>, + /// Overrides the reentry cooldown applied after an eviction. When set, a retry of + /// the evicted playback inside this window is suppressed instead of evicting its + /// replacement. Used by the reentry-suppression scenario. + #[serde(default)] + pub recent_eviction_reentry_ttl_ms: Option, + #[serde(default)] + pub grace: Option, + #[serde(default)] + pub provider_max_connections: Option, + #[serde(default)] + pub expected_provider_slots: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UserPolicy { + pub max_connections: usize, + #[serde(default)] + pub soft_connections: usize, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GraceContract { + pub mode: GraceMode, + pub timeout_millis: u64, +} + +impl PolicyContract { + #[must_use] + pub fn effective_admission_strategies(&self) -> Vec { + self.admission_strategies.clone().unwrap_or_else(|| { + self.grace.as_ref().map_or_else(Vec::new, |grace| match grace.mode { + GraceMode::Instant => vec![AdmissionStrategy::GraceInstantStream], + GraceMode::HoldStream => vec![AdmissionStrategy::GraceHoldStream], + }) + }) + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TuliproxConfig { + pub base_url: String, + #[serde(default)] + pub api_base_url: Option, + #[serde(default)] + pub admin_username: Option, + #[serde(default)] + pub admin_password_env: Option, + #[serde(default)] + pub playlist_url: Option, + #[serde(default)] + pub bootstrap: Option, + #[serde(default)] + pub execution_mode: ExecutionMode, + #[serde(default)] + pub playback_endpoint: PlaybackEndpoint, + #[serde(default)] + pub fixture_stream: FixtureStreamOptions, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OriginConfig { + pub run_id: String, + #[serde(default)] + pub account_limit: Option, + #[serde(default)] + pub limit_mode: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BootstrapConfig { + pub command: String, + #[serde(default)] + pub arguments: Vec, + pub readiness_url: String, + #[serde(default = "default_readiness_timeout_millis")] + pub readiness_timeout_millis: u64, +} + +fn default_readiness_timeout_millis() -> u64 { 30_000 } + +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionMode { + #[default] + IsolatedFixture, + ExistingInstance, +} + +/// Selects which Tuliprox playback endpoint the controller uses to build stream URLs. +/// `m3u` (default) discovers URLs from the M3U playlist; `xtream` builds an Xtream live URL +/// from the discovered virtual ID. Currently only `xtream_ts` channels support `xtream`. +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PlaybackEndpoint { + #[default] + M3u, + Xtream, +} + +/// Controls per-fixture live-stream sharing. Defaults mirror the current global behaviour +/// (both HLS and MPEG-TS sharing enabled) so existing scenarios are unaffected. +#[derive(Debug, Clone, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct FixtureStreamOptions { + pub share_live_hls: bool, + pub share_live_mpeg_ts: bool, +} + +impl Default for FixtureStreamOptions { + fn default() -> Self { Self { share_live_hls: true, share_live_mpeg_ts: true } } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Actor { + pub id: String, + pub agent: String, + #[serde(default)] + pub username: Option, + #[serde(default)] + pub password_env: Option, + #[serde(default)] + pub user_agent: Option, + #[serde(default)] + pub client_ip: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClientIp { + pub mode: String, + #[serde(default)] + pub value: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Channel { + pub origin_marker: u32, + pub protocol: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Step { + pub command_id: String, + #[serde(default)] + pub expect: ExpectedPlayback, + pub start: Option, + #[serde(default)] + pub stop: Option, + #[serde(default)] + pub repeat: Option, + #[serde(default)] + pub await_frames: Option, + #[serde(default, rename = "await")] + pub await_condition: Option, + #[serde(default)] + pub assert_origin: Option, + #[serde(default)] + pub assert_runtime: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ExpectedRejection { + #[serde(default)] + pub custom_video: Option, + #[serde(default)] + pub http_status: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExpectedPlayback { + #[default] + Streaming, + Rejected, + RejectedWith(ExpectedRejection), + /// A recently evicted playback retried and the SUT terminated it quietly + /// (HTTP 204, no custom error video, no connection-denied event). + Suppressed, +} + +impl ExpectedPlayback { + #[must_use] + pub fn is_streaming(&self) -> bool { matches!(self, Self::Streaming) } + + #[must_use] + pub fn is_rejected(&self) -> bool { matches!(self, Self::Rejected | Self::RejectedWith(_) | Self::Suppressed) } + + #[must_use] + pub fn matches_outcome(&self, outcome: &crate::protocol::PlaybackOutcome) -> bool { + use crate::protocol::{PlaybackOutcome, RejectionReason}; + match (self, outcome) { + (Self::Streaming, PlaybackOutcome::Streaming { .. }) + | (Self::Rejected, PlaybackOutcome::AdmissionRejected { .. }) + // A quiet suppression is exactly the 204 the SUT returns for a + // reentry-protected retry. Any other rejection reason must fail. + | (Self::Suppressed, PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(204) }) => { + true + } + (Self::RejectedWith(expected), PlaybackOutcome::AdmissionRejected { reason }) => match reason { + RejectionReason::CustomVideo(kind) => { + expected.custom_video.is_none_or(|expected_kind| expected_kind == *kind) + } + RejectionReason::HttpStatus(status) => { + expected.http_status.is_none_or(|expected_status| expected_status == *status) + } + _ => expected.custom_video.is_none() && expected.http_status.is_none(), + }, + _ => false, + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AwaitCondition { + #[serde(default)] + pub valid_frames: Option, +} + +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct AssertOrigin { + #[serde(default)] + pub max_active_tcp: Option, + #[serde(default)] + pub max_active_body: Option, + #[serde(default)] + pub active_body: Option, + #[serde(default)] + pub no_evictions: Option, + #[serde(default)] + pub no_limit_rejections: Option, +} + +/// Per-step assertions against the SUT runtime status (`/api/v1/status`). +/// +/// These verify the user-side accounting the operator sees in the Web UI, so a +/// leaked or double-counted connection fails the scenario even when the origin +/// and provider counters look correct. +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct RuntimeAssertions { + #[serde(default)] + pub active_users: Option, + #[serde(default)] + pub active_user_connections: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum PostReadAction { + #[default] + KeepOpen, + Pause, + Close, +} + +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct Start { + pub actor: String, + pub playback_id: String, + pub session_group: String, + #[serde(default)] + pub url: Option, + #[serde(default)] + pub channel: Option, + #[serde(default)] + pub method: Option, + #[serde(default)] + pub range: Option, + #[serde(default)] + pub expected_status: Option, + #[serde(default)] + pub expected_content_range: Option, + #[serde(default)] + pub read_limit_bytes: Option, + #[serde(default)] + pub post_read_action: Option, + #[serde(default)] + pub user_agent: Option, + #[serde(default)] + pub vod_object: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Stop { + pub playback_id: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Repeat { + pub count: usize, + pub steps: Vec, +} + +impl Scenario { + pub fn from_path(path: &Path) -> Result { + let content = fs::read_to_string(path)?; + let scenario: Self = + serde_saphyr::from_str(&content).map_err(|error| TestkitError::Configuration(error.to_string()))?; + scenario.validate()?; + Ok(scenario) + } + + pub fn validate(&self) -> Result<(), TestkitError> { + if self.schema_version != 1 { + return Err(TestkitError::Configuration("unsupported scenario schema version".to_owned())); + } + if self.name.trim().is_empty() || self.tuliprox.base_url.trim().is_empty() { + return Err(TestkitError::Configuration("scenario name and Tuliprox URL are required".to_owned())); + } + if self.tuliprox.execution_mode == ExecutionMode::IsolatedFixture && self.tuliprox.bootstrap.is_none() { + return Err(TestkitError::Configuration("isolated_fixture requires a bootstrap definition".to_owned())); + } + if self.origin.as_ref().is_some_and(|origin| origin.run_id.trim().is_empty()) { + return Err(TestkitError::Configuration("origin run_id cannot be empty".to_owned())); + } + if self.policy_contract.as_ref().and_then(|contract| contract.provider_max_connections) == Some(0) { + return Err(TestkitError::Configuration( + "provider_max_connections must be positive when specified".to_owned(), + )); + } + let actor_ids = self.actors.iter().map(|actor| actor.id.as_str()).collect::>(); + if actor_ids.len() != self.actors.len() || actor_ids.iter().any(|id| id.is_empty()) { + return Err(TestkitError::Configuration("actor IDs must be unique and non-empty".to_owned())); + } + let steps = self.expanded_steps()?; + let mut command_ids = HashSet::new(); + let mut playback_ids = HashSet::new(); + let mut seen_playback_ids = HashSet::new(); + for step in &steps { + if !command_ids.insert(step.command_id.as_str()) || step.command_id.is_empty() { + return Err(TestkitError::Configuration("command IDs must be unique and non-empty".to_owned())); + } + if step.start.is_some() == step.stop.is_some() { + return Err(TestkitError::Configuration( + "each step must contain exactly one of start or stop".to_owned(), + )); + } + let Some(start) = &step.start else { + let stop = step.stop.as_ref().ok_or_else(|| TestkitError::Configuration("missing stop".to_owned()))?; + if stop.playback_id.is_empty() || !playback_ids.contains(stop.playback_id.as_str()) { + return Err(TestkitError::Configuration(format!( + "stop references inactive playback {}", + stop.playback_id + ))); + } + playback_ids.remove(stop.playback_id.as_str()); + continue; + }; + if !actor_ids.contains(start.actor.as_str()) { + return Err(TestkitError::Configuration(format!("unknown actor {}", start.actor))); + } + if start.playback_id.is_empty() || start.session_group.is_empty() { + return Err(TestkitError::Configuration("start requires playback_id and session_group".to_owned())); + } + let target_count = [start.url.is_some(), start.channel.is_some(), start.vod_object.is_some()] + .into_iter() + .filter(|&v| v) + .count(); + if target_count != 1 { + return Err(TestkitError::Configuration( + "start requires exactly one of url, channel, or vod_object".to_owned(), + )); + } + if start.url.as_deref().is_some_and(str::is_empty) + || start.channel.as_deref().is_some_and(str::is_empty) + || start.vod_object.as_deref().is_some_and(str::is_empty) + { + return Err(TestkitError::Configuration( + "start URL, channel, or vod_object cannot be empty".to_owned(), + )); + } + if let Some(channel) = &start.channel { + if !self.channels.contains_key(channel) { + return Err(TestkitError::Configuration(format!("unknown channel {channel}"))); + } + } + if !seen_playback_ids.insert(start.playback_id.as_str()) { + return Err(TestkitError::Configuration(format!("duplicate playback ID {}", start.playback_id))); + } + playback_ids.insert(start.playback_id.as_str()); + if step.expect.is_rejected() + && step + .await_frames + .or_else(|| step.await_condition.as_ref().and_then(|condition| condition.valid_frames)) + .is_some() + { + return Err(TestkitError::Configuration("rejected playback must not await valid frames".to_owned())); + } + } + self.validate_policy_contract()?; + self.validate_playback_endpoint()?; + Ok(()) + } + + fn validate_policy_contract(&self) -> Result<(), TestkitError> { + if let Some(contract) = &self.policy_contract { + if !contract.user_access_control && !contract.users.is_empty() { + return Err(TestkitError::Configuration("policy users require user_access_control".to_owned())); + } + if contract.grace.as_ref().is_some_and(|grace| grace.timeout_millis == 0) { + return Err(TestkitError::Configuration("grace timeout must be non-zero".to_owned())); + } + } + Ok(()) + } + + fn validate_playback_endpoint(&self) -> Result<(), TestkitError> { + if self.tuliprox.playback_endpoint == PlaybackEndpoint::Xtream { + for (name, channel) in &self.channels { + if channel.protocol != "xtream_ts" { + return Err(TestkitError::Configuration(format!( + "channel {name} uses protocol '{}' which is not supported with playback_endpoint: xtream (only xtream_ts is supported)", + channel.protocol + ))); + } + } + } + Ok(()) + } + + pub fn expanded_steps(&self) -> Result, TestkitError> { + let mut expanded = Vec::new(); + expand_steps(&self.steps, "", &mut expanded)?; + Ok(expanded) + } +} + +fn expand_steps(steps: &[Step], prefix: &str, expanded: &mut Vec) -> Result<(), TestkitError> { + const MAX_REPEAT_COUNT: usize = 1_000; + for step in steps { + if step.command_id.is_empty() { + return Err(TestkitError::Configuration("command IDs must be unique and non-empty".to_owned())); + } + if let Some(repeat) = &step.repeat { + if step.start.is_some() || step.stop.is_some() { + return Err(TestkitError::Configuration("repeat cannot be combined with start or stop".to_owned())); + } + if repeat.count == 0 || repeat.count > MAX_REPEAT_COUNT || repeat.steps.is_empty() { + return Err(TestkitError::Configuration("repeat requires 1 through 1000 child steps".to_owned())); + } + for iteration in 0..repeat.count { + expand_steps(&repeat.steps, &format!("{prefix}{}:{iteration}:", step.command_id), expanded)?; + } + continue; + } + if step.start.is_some() == step.stop.is_some() { + return Err(TestkitError::Configuration("each step must contain exactly one of start or stop".to_owned())); + } + let mut child = step.clone(); + child.command_id = format!("{prefix}{}", step.command_id); + if let Some(start) = child.start.as_mut() { + start.playback_id = format!("{prefix}{}", start.playback_id); + start.session_group = format!("{prefix}{}", start.session_group); + } + if let Some(stop) = child.stop.as_mut() { + stop.playback_id = format!("{prefix}{}", stop.playback_id); + } + expanded.push(child); + } + Ok(()) +} + +#[must_use] +pub fn lease_duration(millis: u64) -> Duration { Duration::from_millis(millis.max(1)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn duplicate_playbacks_are_rejected() { + let scenario = Scenario { + schema_version: 1, + name: "test".to_owned(), + tuliprox: TuliproxConfig { + base_url: "http://example.invalid".to_owned(), + api_base_url: None, + admin_username: None, + admin_password_env: None, + playlist_url: None, + bootstrap: None, + execution_mode: ExecutionMode::IsolatedFixture, + playback_endpoint: PlaybackEndpoint::default(), + fixture_stream: FixtureStreamOptions::default(), + }, + origin: None, + actors: vec![Actor { + id: "a".to_owned(), + agent: "local".to_owned(), + username: None, + password_env: None, + user_agent: None, + client_ip: None, + }], + channels: HashMap::new(), + agents: AgentsConfig::default(), + policy_contract: None, + steps: (0..2) + .map(|number| Step { + command_id: number.to_string(), + expect: ExpectedPlayback::Streaming, + start: Some(Start { + actor: "a".to_owned(), + playback_id: "same".to_owned(), + session_group: "s".to_owned(), + url: Some("http://example.invalid".to_owned()), + channel: None, + ..Default::default() + }), + stop: None, + repeat: None, + await_frames: None, + await_condition: None, + assert_origin: None, + assert_runtime: None, + }) + .collect(), + }; + assert!(scenario.validate().is_err()); + } + + #[test] + fn checked_in_admission_scenarios_match_the_versioned_schema() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test/fixtures/testkit/scenarios"); + for filename in [ + "hard-limit.yml", + "evict-same-ip-latest.yml", + "evict-same-ip-oldest.yml", + "evict-user-oldest.yml", + "evict-user-latest.yml", + "provider-hard-limit.yml", + "shared-stream-single-provider-slot.yml", + "vod-range-reopen-preserves-three-live.yml", + "vod-range-reopen-strict-cap.yml", + "vod-reopen-backpressured-body.yml", + "live-ts-same-channel-retry-latest-wins.yml", + "reentry-suppresses-evicted-retry.yml", + "soft-slot-precedes-eviction.yml", + "soft-slot-exhausts-without-upstream-leak.yml", + ] { + assert!(Scenario::from_path(&root.join(filename)).is_ok(), "{filename}"); + } + } + + #[test] + fn stop_must_reference_an_active_playback() { + let scenario: Scenario = serde_saphyr::from_str( + r" +schema_version: 1 +name: stop-validation +tuliprox: { base_url: http://example.invalid, execution_mode: existing_instance } +actors: [{ id: a, agent: local }] +steps: + - command_id: stop-missing + stop: { playback_id: missing } +", + ) + .unwrap(); + assert!(scenario.validate().is_err()); + } + + #[test] + fn repeat_derives_distinct_playback_and_command_ids_per_iteration() { + let scenario: Scenario = serde_saphyr::from_str( + r" +schema_version: 1 +name: repeat-validation +tuliprox: { base_url: http://example.invalid, execution_mode: existing_instance } +actors: [{ id: a, agent: local }] +steps: + - command_id: repeat + repeat: + count: 2 + steps: + - command_id: begin + start: { actor: a, playback_id: playback, session_group: session, url: http://example.invalid/17.ts } + - command_id: end + stop: { playback_id: playback } +", + ) + .unwrap(); + assert!(scenario.validate().is_ok()); + let expanded = scenario.expanded_steps().unwrap(); + assert_eq!(expanded.len(), 4); + assert_eq!(expanded[0].command_id, "repeat:0:begin"); + assert_eq!(expanded[0].start.as_ref().map(|start| start.playback_id.as_str()), Some("repeat:0:playback")); + assert_eq!(expanded[2].command_id, "repeat:1:begin"); + assert_eq!(expanded[3].stop.as_ref().map(|stop| stop.playback_id.as_str()), Some("repeat:1:playback")); + } + + #[test] + fn rejected_playback_does_not_match_500_corrupt_data_or_early_eof() { + use crate::{ + custom_video::CustomVideoKind, + protocol::{PlaybackOutcome, RejectionReason}, + }; + + let expected_rejected = ExpectedPlayback::Rejected; + + // Proved admission rejection matches + let valid_cv = PlaybackOutcome::AdmissionRejected { + reason: RejectionReason::CustomVideo(CustomVideoKind::UserConnectionsExhausted), + }; + assert!(expected_rejected.matches_outcome(&valid_cv)); + + let valid_http = PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(429) }; + assert!(expected_rejected.matches_outcome(&valid_http)); + + // HTTP 500 MUST NOT pass as expected rejection + let http_500 = PlaybackOutcome::HttpError { status: 500 }; + assert!(!expected_rejected.matches_outcome(&http_500)); + + // Invalid marker/corrupt frame MUST NOT pass + let invalid_marker = PlaybackOutcome::InvalidData { message: "wrong marker".to_owned() }; + assert!(!expected_rejected.matches_outcome(&invalid_marker)); + + // Early EOF MUST NOT pass + let early_eof = PlaybackOutcome::UnexpectedEof { frames: 0, bytes: 0 }; + assert!(!expected_rejected.matches_outcome(&early_eof)); + + // Transport error MUST NOT pass + let transport_err = PlaybackOutcome::TransportError { message: "connection reset".to_owned() }; + assert!(!expected_rejected.matches_outcome(&transport_err)); + + // Idle timeout MUST NOT pass + assert!(!expected_rejected.matches_outcome(&PlaybackOutcome::IdleTimeout)); + } + + #[test] + fn suppressed_matches_only_204_quiet_termination() { + use crate::{ + custom_video::CustomVideoKind, + protocol::{PlaybackOutcome, RejectionReason}, + }; + + let suppressed = ExpectedPlayback::Suppressed; + assert!(suppressed + .matches_outcome(&PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(204) })); + // A custom error video (the regression this scenario guards against) must fail. + assert!(!suppressed.matches_outcome(&PlaybackOutcome::AdmissionRejected { + reason: RejectionReason::CustomVideo(CustomVideoKind::UserConnectionsExhausted), + })); + assert!(!suppressed + .matches_outcome(&PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(503) })); + // An empty body without the 204 classification (bare EOF) must fail too. + assert!(!suppressed.matches_outcome(&PlaybackOutcome::UnexpectedEof { frames: 0, bytes: 0 })); + } + + #[test] + fn playback_endpoint_defaults_to_m3u() { + let cfg: TuliproxConfig = serde_saphyr::from_str("base_url: 'http://example.invalid'").unwrap(); + assert_eq!(cfg.playback_endpoint, PlaybackEndpoint::M3u); + } + + #[test] + fn playback_endpoint_xtream_deserializes() { + let cfg: TuliproxConfig = + serde_saphyr::from_str("base_url: 'http://example.invalid'\nplayback_endpoint: xtream").unwrap(); + assert_eq!(cfg.playback_endpoint, PlaybackEndpoint::Xtream); + } + + #[test] + fn fixture_stream_defaults_enable_sharing() { + let opts = FixtureStreamOptions::default(); + assert!(opts.share_live_hls); + assert!(opts.share_live_mpeg_ts); + } + + #[test] + fn playback_endpoint_xtream_rejects_non_xtream_ts_channel() { + let scenario: Scenario = serde_saphyr::from_str( + r" +schema_version: 1 +name: xtream-validation +tuliprox: + base_url: 'http://example.invalid' + execution_mode: existing_instance + playback_endpoint: xtream +actors: [{ id: a, agent: local }] +channels: + ch: { origin_marker: 17, protocol: hls } +steps: + - command_id: s + start: { actor: a, playback_id: p, session_group: s, channel: ch } +", + ) + .unwrap(); + assert!(scenario.validate().is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/control.rs b/tools/tuliprox-testkit/src/control.rs new file mode 100644 index 000000000..ec0368166 --- /dev/null +++ b/tools/tuliprox-testkit/src/control.rs @@ -0,0 +1,63 @@ +use crate::{ + protocol::{PlaybackId, RunId}, + TestkitError, +}; +use std::collections::HashMap; + +/// Tracks the semantic start identity for a run independently from transport message IDs. +/// A replay with identical parameters is harmless; a conflicting replay is rejected. +#[derive(Debug, Default)] +pub struct PlaybackRegistry { + starts: HashMap<(RunId, u64, PlaybackId), String>, +} + +impl PlaybackRegistry { + pub fn register_start( + &mut self, + run_id: RunId, + generation: u64, + playback_id: PlaybackId, + fingerprint: String, + ) -> Result { + let key = (run_id, generation, playback_id); + match self.starts.get(&key) { + None => { + self.starts.insert(key, fingerprint); + Ok(StartDisposition::New) + } + Some(existing) if existing == &fingerprint => Ok(StartDisposition::Replay), + Some(_) => Err(TestkitError::Protocol("playback ID was reused with different start parameters".to_owned())), + } + } + + pub fn stop(&mut self, run_id: &RunId, generation: u64, playback_id: &PlaybackId) -> bool { + self.starts.remove(&(run_id.clone(), generation, playback_id.clone())).is_some() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StartDisposition { + New, + Replay, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replay_is_safe_but_conflict_is_rejected() { + let mut registry = PlaybackRegistry::default(); + let run = RunId::new("run"); + let playback = PlaybackId::new("playback"); + assert!(matches!( + registry.register_start(run.clone(), 1, playback.clone(), "first".to_owned()), + Ok(StartDisposition::New) + )); + assert!(matches!( + registry.register_start(run.clone(), 1, playback.clone(), "first".to_owned()), + Ok(StartDisposition::Replay) + )); + assert!(registry.register_start(run, 1, playback, "different".to_owned()).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/custom_video.rs b/tools/tuliprox-testkit/src/custom_video.rs new file mode 100644 index 000000000..fec373bd0 --- /dev/null +++ b/tools/tuliprox-testkit/src/custom_video.rs @@ -0,0 +1,124 @@ +use crate::TestkitError; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::Path}; + +const PREFIX: &[u8] = b"TPX-CVS-"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CustomVideoKind { + ChannelUnavailable, + UserConnectionsExhausted, + ProviderConnectionsExhausted, + LowPriorityPreempted, + UserAccountExpired, + PanelApiProvisioning, + HlsSessionOrLeaseExpired, +} + +impl CustomVideoKind { + fn parse(value: &str) -> Option { + match value { + "CHANNEL_UNAVAILABLE" => Some(Self::ChannelUnavailable), + "USER_CONNECTIONS_EXHAUSTED" => Some(Self::UserConnectionsExhausted), + "PROVIDER_CONNECTIONS_EXHAUSTED" => Some(Self::ProviderConnectionsExhausted), + "LOW_PRIORITY_PREEMPTED" => Some(Self::LowPriorityPreempted), + "USER_ACCOUNT_EXPIRED" => Some(Self::UserAccountExpired), + "PANEL_API_PROVISIONING" => Some(Self::PanelApiProvisioning), + "HLS_SESSION_OR_LEASE_EXPIRED" => Some(Self::HlsSessionOrLeaseExpired), + _ => None, + } + } +} + +#[derive(Debug, Default)] +pub struct WireMarkerScanner { + retained: Vec, +} + +impl WireMarkerScanner { + pub fn push(&mut self, bytes: &[u8]) -> Result, TestkitError> { + self.retained.extend_from_slice(bytes); + if let Some(position) = self.retained.windows(PREFIX.len()).position(|window| window == PREFIX) { + let value_start = position + PREFIX.len(); + let Some(value_end) = + self.retained[value_start..].iter().position(|byte| !matches!(byte, b'A'..=b'Z' | b'_')) + else { + self.limit_buffer(); + return Ok(None); + }; + let marker = std::str::from_utf8(&self.retained[value_start..value_start + value_end]) + .map_err(|error| TestkitError::Protocol(format!("invalid custom-video marker: {error}")))?; + let kind = CustomVideoKind::parse(marker) + .ok_or_else(|| TestkitError::Protocol(format!("unknown custom-video marker {marker}")))?; + self.retained.clear(); + return Ok(Some(kind)); + } + self.limit_buffer(); + Ok(None) + } + + fn limit_buffer(&mut self) { + const MAX_RETAINED: usize = 128; + if self.retained.len() > MAX_RETAINED { + let start = self.retained.len() - MAX_RETAINED; + self.retained.drain(..start); + } + } +} + +#[must_use] +pub fn fixture_digest(bytes: &[u8]) -> String { blake3::hash(bytes).to_hex().to_string() } + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FixtureManifest { + pub schema_version: u16, + pub fixtures: BTreeMap, +} + +/// Verifies fixture installation bytes only. It must never be used to classify +/// bytes observed after Tuliprox has processed a transport stream. +pub fn verify_fixture_manifest(root: &Path, manifest: &FixtureManifest) -> Result<(), TestkitError> { + if manifest.schema_version != 1 { + return Err(TestkitError::Configuration("unsupported custom-video fixture manifest schema".to_owned())); + } + for (relative_path, expected_digest) in &manifest.fixtures { + let path = Path::new(relative_path); + if path.is_absolute() || path.components().any(|component| matches!(component, std::path::Component::ParentDir)) + { + return Err(TestkitError::Configuration(format!("unsafe fixture path {relative_path}"))); + } + let bytes = std::fs::read(root.join(path))?; + let actual_digest = fixture_digest(&bytes); + if actual_digest != *expected_digest { + return Err(TestkitError::Protocol(format!("fixture digest mismatch for {relative_path}"))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recognizes_marker_split_across_transport_chunks() { + let mut scanner = WireMarkerScanner::default(); + assert!(scanner.push(b"ignored TPX-CVS-USER_").unwrap().is_none()); + assert_eq!(scanner.push(b"CONNECTIONS_EXHAUSTED;").unwrap(), Some(CustomVideoKind::UserConnectionsExhausted)); + } + + #[test] + fn fixture_digest_is_deterministic() { + assert_eq!(fixture_digest(b"fixture"), fixture_digest(b"fixture")); + } + + #[test] + fn checked_in_fixture_manifest_matches_installation_bytes() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test/fixtures"); + let manifest_path = root.join("testkit/custom-video-fixtures.json"); + let manifest: FixtureManifest = serde_json::from_slice(&std::fs::read(manifest_path).unwrap()).unwrap(); + assert!(verify_fixture_manifest(&root, &manifest).is_ok()); + } +} diff --git a/tools/tuliprox-testkit/src/discovery.rs b/tools/tuliprox-testkit/src/discovery.rs new file mode 100644 index 000000000..d162eca45 --- /dev/null +++ b/tools/tuliprox-testkit/src/discovery.rs @@ -0,0 +1,146 @@ +use crate::TestkitError; +use std::collections::HashMap; + +/// Maps a stable synthetic-origin marker to the virtual playback URL published by Tuliprox. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VirtualIdMap { + urls: HashMap, + named_urls: HashMap, +} + +impl VirtualIdMap { + pub fn from_m3u(document: &str) -> Result { + let mut urls = HashMap::new(); + let mut named_urls = HashMap::new(); + let mut pending_marker = None; + let mut pending_name = None; + for raw_line in document.lines() { + let line = raw_line.trim(); + if line.starts_with("#EXTINF:") { + pending_marker = marker_from_extinf(line); + pending_name = name_from_extinf(line); + continue; + } + if line.is_empty() || line.starts_with('#') { + continue; + } + if let Some(marker) = pending_marker.take() { + if urls.insert(marker, line.to_owned()).is_some() { + return Err(TestkitError::Configuration(format!("duplicate origin marker {marker} in playlist"))); + } + } + if let Some(name) = pending_name.take() { + if named_urls.insert(name.clone(), line.to_owned()).is_some() { + return Err(TestkitError::Configuration(format!("duplicate named media {name} in playlist"))); + } + } + } + if urls.is_empty() && named_urls.is_empty() { + return Err(TestkitError::Configuration("playlist has no testkit marker entries".to_owned())); + } + Ok(Self { urls, named_urls }) + } + + pub fn playback_url(&self, marker: u32) -> Result<&str, TestkitError> { + self.urls + .get(&marker) + .map(String::as_str) + .ok_or_else(|| TestkitError::Configuration(format!("origin marker {marker} was not published by Tuliprox"))) + } + + pub fn named_playback_url(&self, name: &str) -> Result<&str, TestkitError> { + self.named_urls + .get(name) + .map(String::as_str) + .ok_or_else(|| TestkitError::Configuration(format!("named media {name} was not published by Tuliprox"))) + } + + /// Extract the numeric virtual ID from the URL stored for `marker`. The virtual ID is the last + /// path segment after stripping any query string and file extension. Returns an error if the + /// segment cannot be parsed as a `u64`. + pub fn virtual_id_from_marker(&self, marker: u32) -> Result { + let url = self.playback_url(marker)?; + extract_virtual_id_from_url(url) + } +} + +fn marker_from_extinf(line: &str) -> Option { + let value = line.split("tvg-id=\"").nth(1)?.split('"').next()?; + value.strip_prefix("test-")?.parse().ok() +} + +fn name_from_extinf(line: &str) -> Option { + let value = line.split("tvg-id=\"").nth(1)?.split('"').next()?; + value.strip_prefix("test-vod-").map(str::to_owned) +} + +/// Extract the numeric virtual ID from the last path segment of a URL. +/// Query strings and file extensions (e.g. `.ts`, `.m3u8`) are stripped before parsing. +fn extract_virtual_id_from_url(url: &str) -> Result { + // Drop query string + let path = url.split('?').next().unwrap_or(url); + // Drop file extension: find the last `.` in the last path segment only + let path = { + let last_slash = path.rfind('/').map_or(0, |i| i + 1); + if let Some(dot_pos) = path[last_slash..].rfind('.') { + &path[..last_slash + dot_pos] + } else { + path + } + }; + let segment = path.rsplit('/').next().unwrap_or(""); + segment.parse::().map_err(|_| { + TestkitError::Configuration(format!( + "cannot extract numeric virtual ID from URL {url}: last path segment {segment:?} is not a valid u64" + )) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovers_virtual_playback_urls_without_assuming_their_ids() { + let playlist = "#EXTM3U\n#EXTINF:-1 tvg-id=\"test-17\",News\nhttp://tuliprox/m3u/abc/17\n#EXTINF:-1 tvg-id=\"test-vod-movie.mkv\" type=\"movie\",Movie\nhttp://tuliprox/movie/abc/def/1.mkv\n"; + let map = VirtualIdMap::from_m3u(playlist).unwrap(); + assert_eq!(map.playback_url(17).unwrap(), "http://tuliprox/m3u/abc/17"); + assert_eq!(map.named_playback_url("movie.mkv").unwrap(), "http://tuliprox/movie/abc/def/1.mkv"); + } + + #[test] + fn extracts_virtual_id_from_plain_m3u_url() { + let map = VirtualIdMap::from_m3u("#EXTM3U\n#EXTINF:-1 tvg-id=\"test-17\",News\nhttp://tuliprox/m3u/abc/42\n") + .unwrap(); + assert_eq!(map.virtual_id_from_marker(17).unwrap(), 42); + } + + #[test] + fn strips_query_string_before_extracting_virtual_id() { + let map = VirtualIdMap::from_m3u( + "#EXTM3U\n#EXTINF:-1 tvg-id=\"test-5\",Ch5\nhttp://tuliprox/m3u/user/99?token=abc\n", + ) + .unwrap(); + assert_eq!(map.virtual_id_from_marker(5).unwrap(), 99); + } + + #[test] + fn strips_file_extension_before_extracting_virtual_id() { + let map = VirtualIdMap::from_m3u("#EXTM3U\n#EXTINF:-1 tvg-id=\"test-3\",Ch3\nhttp://tuliprox/m3u/user/77.ts\n") + .unwrap(); + assert_eq!(map.virtual_id_from_marker(3).unwrap(), 77); + } + + #[test] + fn non_numeric_last_segment_is_an_error() { + let map = VirtualIdMap::from_m3u("#EXTM3U\n#EXTINF:-1 tvg-id=\"test-1\",Ch1\nhttp://tuliprox/m3u/user/abc\n") + .unwrap(); + assert!(map.virtual_id_from_marker(1).is_err()); + } + + #[test] + fn duplicate_named_media_is_rejected() { + let playlist = "#EXTM3U\n#EXTINF:-1 tvg-id=\"test-vod-movie.mkv\",Movie\nhttp://tuliprox/movie/abc/def/1.mkv\n#EXTINF:-1 tvg-id=\"test-vod-movie.mkv\",Movie Again\nhttp://tuliprox/movie/abc/def/2.mkv\n"; + assert!(matches!(VirtualIdMap::from_m3u(playlist), Err(TestkitError::Configuration(_)))); + } +} diff --git a/tools/tuliprox-testkit/src/faults.rs b/tools/tuliprox-testkit/src/faults.rs new file mode 100644 index 000000000..30275a896 --- /dev/null +++ b/tools/tuliprox-testkit/src/faults.rs @@ -0,0 +1,50 @@ +use crate::TestkitError; +use std::collections::HashMap; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OriginFault { + CloseBeforeFirstByte, + CloseAfterBytes(u64), + StallAfterBytes(u64), + CorruptNextFrame, + UnexpectedMarker(u32), +} + +#[derive(Debug, Default)] +pub struct FaultSchedule { + faults: HashMap, +} + +impl FaultSchedule { + pub fn insert(&mut self, fault_id: String, fault: OriginFault) -> Result<(), TestkitError> { + if self.faults.contains_key(&fault_id) { + return Err(TestkitError::Configuration(format!("fault ID {fault_id} already exists"))); + } + self.faults.insert(fault_id, fault); + Ok(()) + } + + pub fn take(&mut self, fault_id: &str) -> Option { self.faults.remove(fault_id) } + + pub fn clear(&mut self, fault_id: &str) -> Result<(), TestkitError> { + self.faults + .remove(fault_id) + .map(|_| ()) + .ok_or_else(|| TestkitError::Configuration(format!("fault ID {fault_id} does not exist"))) + } + + pub fn clear_all(&mut self) { self.faults.clear(); } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fault_is_consumed_only_once() { + let mut schedule = FaultSchedule::default(); + schedule.insert("close".to_owned(), OriginFault::CloseBeforeFirstByte).unwrap(); + assert_eq!(schedule.take("close"), Some(OriginFault::CloseBeforeFirstByte)); + assert_eq!(schedule.take("close"), None); + } +} diff --git a/tools/tuliprox-testkit/src/frame.rs b/tools/tuliprox-testkit/src/frame.rs new file mode 100644 index 000000000..49c43eb54 --- /dev/null +++ b/tools/tuliprox-testkit/src/frame.rs @@ -0,0 +1,251 @@ +use crate::{ + protocol::{OriginStreamId, RunId}, + TestkitError, +}; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use crc32fast::hash; + +pub const MAGIC: [u8; 8] = *b"TPXTEST\0"; +const VERSION: u8 = 1; +const FIXED_HEADER_LEN: usize = 8 + 1 + 1 + 2 + 16 + 16 + 4 + 8 + 8 + 4 + 4 + 4; +const FIXED_HEADER_LEN_U16: u16 = 76; +const TRAILER_LEN: usize = 4; +pub const MAX_PAYLOAD_LEN: usize = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + pub run_id: [u8; 16], + pub origin_stream_id: [u8; 16], + pub channel_marker: u32, + pub sequence: u64, + pub origin_mono_nanos: u64, + pub configured_bps: u32, + pub payload: Bytes, +} + +impl Frame { + #[must_use] + pub fn synthetic(run_id: &RunId, stream_id: &OriginStreamId, marker: u32, sequence: u64) -> Self { + let mut run = [0_u8; 16]; + let mut stream = [0_u8; 16]; + run.copy_from_slice(&blake3::hash(run_id.0.as_bytes()).as_bytes()[..16]); + stream.copy_from_slice(&blake3::hash(stream_id.0.as_bytes()).as_bytes()[..16]); + let payload = format!("marker={marker};sequence={sequence}\n").into_bytes(); + Self { + run_id: run, + origin_stream_id: stream, + channel_marker: marker, + sequence, + origin_mono_nanos: 0, + configured_bps: 64_000, + payload: Bytes::from(payload), + } + } + + pub fn encode(&self, output: &mut BytesMut) -> Result<(), TestkitError> { + let payload_len = self.payload.len(); + if payload_len > MAX_PAYLOAD_LEN { + return Err(TestkitError::Protocol("frame payload exceeds configured limit".to_owned())); + } + let start = output.len(); + output.reserve(FIXED_HEADER_LEN + payload_len + TRAILER_LEN); + output.put_slice(&MAGIC); + output.put_u8(VERSION); + output.put_u8(0); + output.put_u16(FIXED_HEADER_LEN_U16); + output.put_slice(&self.run_id); + output.put_slice(&self.origin_stream_id); + output.put_u32(self.channel_marker); + output.put_u64(self.sequence); + output.put_u64(self.origin_mono_nanos); + output.put_u32(self.configured_bps); + output.put_u32( + u32::try_from(payload_len) + .map_err(|_| TestkitError::Protocol("frame payload length cannot be encoded".to_owned()))?, + ); + let header_crc = hash(&output[start..]); + output.put_u32(header_crc); + output.put_slice(&self.payload); + output.put_u32(hash(&self.payload)); + Ok(()) + } +} + +#[derive(Debug, Default)] +pub struct FrameDecoder { + buffer: BytesMut, +} + +/// Verifies that decoded frames belong to one requested synthetic stream. +/// A late subscriber may start at any sequence, but it can never move backwards +/// or silently switch to another upstream response. +#[derive(Debug)] +pub struct FrameValidator { + expected_run_id: [u8; 16], + expected_marker: u32, + origin_stream_id: Option<[u8; 16]>, + last_sequence: Option, +} + +impl FrameValidator { + #[must_use] + pub fn new(run_id: &RunId, expected_marker: u32) -> Self { + let mut expected_run_id = [0_u8; 16]; + expected_run_id.copy_from_slice(&blake3::hash(run_id.0.as_bytes()).as_bytes()[..16]); + Self { expected_run_id, expected_marker, origin_stream_id: None, last_sequence: None } + } + + pub fn validate(&mut self, frame: &Frame) -> Result<(), TestkitError> { + if frame.run_id != self.expected_run_id { + return Err(TestkitError::Protocol("received a frame for a foreign test run".to_owned())); + } + if frame.channel_marker != self.expected_marker { + return Err(TestkitError::Protocol(format!( + "marker mismatch: expected {}, got {}", + self.expected_marker, frame.channel_marker + ))); + } + if let Some(origin_stream_id) = self.origin_stream_id { + if origin_stream_id != frame.origin_stream_id { + return Err(TestkitError::Protocol("origin stream changed during a playback".to_owned())); + } + } else { + self.origin_stream_id = Some(frame.origin_stream_id); + } + if let Some(previous) = self.last_sequence { + if frame.sequence <= previous { + return Err(TestkitError::Protocol(format!( + "non-increasing frame sequence: previous {previous}, current {}", + frame.sequence + ))); + } + } + let expected_payload = format!("marker={};sequence={}\n", frame.channel_marker, frame.sequence); + if frame.payload.as_ref() != expected_payload.as_bytes() { + return Err(TestkitError::Protocol( + "synthetic frame payload does not match marker and sequence".to_owned(), + )); + } + self.last_sequence = Some(frame.sequence); + Ok(()) + } +} + +impl FrameDecoder { + pub fn push(&mut self, bytes: &[u8]) -> Result, TestkitError> { + self.buffer.extend_from_slice(bytes); + let mut frames = Vec::new(); + loop { + let Some(position) = self.buffer.windows(MAGIC.len()).position(|window| window == MAGIC) else { + self.keep_magic_suffix(); + return Ok(frames); + }; + if position != 0 { + self.buffer.advance(position); + } + if self.buffer.len() < FIXED_HEADER_LEN { + return Ok(frames); + } + let header_len = u16::from_be_bytes([self.buffer[10], self.buffer[11]]) as usize; + if header_len != FIXED_HEADER_LEN || self.buffer[8] != VERSION { + self.buffer.advance(1); + continue; + } + let expected_header_crc = u32::from_be_bytes( + self.buffer[FIXED_HEADER_LEN - 4..FIXED_HEADER_LEN] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid header length".to_owned()))?, + ); + if hash(&self.buffer[..FIXED_HEADER_LEN - 4]) != expected_header_crc { + self.buffer.advance(1); + continue; + } + let payload_len = u32::from_be_bytes( + self.buffer[68..72] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid payload length".to_owned()))?, + ) as usize; + if payload_len > MAX_PAYLOAD_LEN { + self.buffer.advance(1); + continue; + } + let total_len = FIXED_HEADER_LEN + payload_len + TRAILER_LEN; + if self.buffer.len() < total_len { + return Ok(frames); + } + let payload_start = FIXED_HEADER_LEN; + let payload_end = payload_start + payload_len; + let expected_payload_crc = u32::from_be_bytes( + self.buffer[payload_end..total_len] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid trailer length".to_owned()))?, + ); + if hash(&self.buffer[payload_start..payload_end]) != expected_payload_crc { + self.buffer.advance(1); + continue; + } + let mut run_id = [0_u8; 16]; + let mut origin_stream_id = [0_u8; 16]; + run_id.copy_from_slice(&self.buffer[12..28]); + origin_stream_id.copy_from_slice(&self.buffer[28..44]); + let frame = Frame { + run_id, + origin_stream_id, + channel_marker: u32::from_be_bytes( + self.buffer[44..48].try_into().map_err(|_| TestkitError::Protocol("invalid marker".to_owned()))?, + ), + sequence: u64::from_be_bytes( + self.buffer[48..56] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid sequence".to_owned()))?, + ), + origin_mono_nanos: u64::from_be_bytes( + self.buffer[56..64] + .try_into() + .map_err(|_| TestkitError::Protocol("invalid timestamp".to_owned()))?, + ), + configured_bps: u32::from_be_bytes( + self.buffer[64..68].try_into().map_err(|_| TestkitError::Protocol("invalid bitrate".to_owned()))?, + ), + payload: self.buffer[payload_start..payload_end].to_vec().into(), + }; + self.buffer.advance(total_len); + frames.push(frame); + } + } + + fn keep_magic_suffix(&mut self) { + let retained = self.buffer.len().min(MAGIC.len() - 1); + let start = self.buffer.len() - retained; + let suffix = self.buffer.split_off(start); + self.buffer = suffix; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_every_network_split() { + let frame = Frame::synthetic(&RunId::new("run"), &OriginStreamId::new("origin"), 17, 0); + let mut encoded = BytesMut::new(); + frame.encode(&mut encoded).unwrap(); + for split in 0..=encoded.len() { + let mut decoder = FrameDecoder::default(); + let mut frames = decoder.push(&encoded[..split]).unwrap(); + frames.extend(decoder.push(&encoded[split..]).unwrap()); + assert_eq!(frames, vec![frame.clone()]); + } + } + + #[test] + fn validator_rejects_foreign_runs_and_sequence_replay() { + let run = RunId::new("run"); + let stream = OriginStreamId::new("origin"); + let mut validator = FrameValidator::new(&run, 17); + assert!(validator.validate(&Frame::synthetic(&run, &stream, 17, 4)).is_ok()); + assert!(validator.validate(&Frame::synthetic(&run, &stream, 17, 4)).is_err()); + assert!(validator.validate(&Frame::synthetic(&RunId::new("other"), &stream, 17, 5)).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/hls.rs b/tools/tuliprox-testkit/src/hls.rs new file mode 100644 index 000000000..63f14d3de --- /dev/null +++ b/tools/tuliprox-testkit/src/hls.rs @@ -0,0 +1,164 @@ +use crate::{ + frame::{FrameDecoder, FrameValidator}, + protocol::RunId, + TestkitError, +}; +use futures::StreamExt; +use std::collections::{BTreeMap, HashSet}; +use url::Url; + +const SEGMENT_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +const MANIFEST_RELOAD_INTERVAL: std::time::Duration = std::time::Duration::from_millis(500); +const MANIFEST_RELOAD_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); + +pub async fn read_segments( + manifest_url: &str, + expected_run_id: &RunId, + expected_marker: u32, + minimum_frames: u64, + headers: &BTreeMap, +) -> Result { + let client = reqwest::Client::builder().http1_only().build()?; + let base = Url::parse(manifest_url).map_err(|error| TestkitError::Configuration(error.to_string()))?; + let mut frames = 0; + let mut validator = FrameValidator::new(expected_run_id, expected_marker); + // Live manifests are reloaded until enough frames were observed. Segment + // identities are remembered across snapshots so a reload never replays a + // segment that was already decoded. + let mut consumed: HashSet = HashSet::new(); + let deadline = tokio::time::Instant::now() + MANIFEST_RELOAD_DEADLINE; + loop { + let manifest = tokio::time::timeout_at(deadline, async { + request_with_headers(client.get(manifest_url), headers).send().await?.error_for_status()?.text().await + }) + .await + .map_err(|_| TestkitError::Protocol("HLS manifest request exceeded overall deadline".to_owned()))??; + let segments = manifest + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_owned) + .collect::>(); + for segment in segments { + if !consumed.insert(segment.clone()) { + continue; + } + let segment_url = base.join(&segment).map_err(|error| TestkitError::Configuration(error.to_string()))?; + let response = tokio::time::timeout_at(deadline, async { + request_with_headers(client.get(segment_url), headers).send().await?.error_for_status() + }) + .await + .map_err(|_| TestkitError::Protocol("HLS segment request exceeded overall deadline".to_owned()))??; + let mut decoder = FrameDecoder::default(); + let mut body = response.bytes_stream(); + loop { + let chunk = + match tokio::time::timeout_at(deadline, tokio::time::timeout(SEGMENT_IDLE_TIMEOUT, body.next())) + .await + { + Ok(Ok(Some(chunk))) => chunk?, + Ok(Ok(None)) => break, + Ok(Err(_)) => return Err(TestkitError::Protocol("HLS segment idle timeout".to_owned())), + Err(_) => return Err(TestkitError::Protocol("HLS read exceeded overall deadline".to_owned())), + }; + for frame in decoder.push(&chunk)? { + validator.validate(&frame)?; + frames += 1; + } + } + if frames >= minimum_frames { + return Ok(frames); + } + } + if frames >= minimum_frames { + return Ok(frames); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!( + "HLS produced {frames} valid frames, expected at least {minimum_frames}" + ))); + } + if tokio::time::timeout_at(deadline, tokio::time::sleep(MANIFEST_RELOAD_INTERVAL)).await.is_err() { + return Err(TestkitError::Protocol(format!( + "HLS produced {frames} valid frames, expected at least {minimum_frames}" + ))); + } + } +} + +fn request_with_headers( + mut request: reqwest::RequestBuilder, + headers: &BTreeMap, +) -> reqwest::RequestBuilder { + for (name, value) in headers { + request = request.header(name, value); + } + request +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{frame::Frame, protocol::OriginStreamId}; + use axum::{ + body::Body, + extract::OriginalUri, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::get, + Router, + }; + use bytes::BytesMut; + use std::sync::Arc; + use tokio::{net::TcpListener, sync::Mutex}; + + #[test] + fn manifest_entries_exclude_directives() { + let entries = "#EXTM3U\n#EXTINF:1,\n0.ts\n" + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .collect::>(); + assert_eq!(entries, vec!["0.ts"]); + } + + #[tokio::test] + async fn forwards_actor_headers_to_manifest_and_segments() -> Result<(), TestkitError> { + let run_id = RunId::new("hls-auth"); + let mut payload = BytesMut::new(); + Frame::synthetic(&run_id, &OriginStreamId::new("segment"), 17, 0).encode(&mut payload)?; + let payload = Arc::new(payload.freeze()); + let requests = Arc::new(Mutex::new(Vec::new())); + let app = Router::new().fallback(get({ + let payload = Arc::clone(&payload); + let requests = Arc::clone(&requests); + move |headers: HeaderMap, uri: OriginalUri| { + let payload = Arc::clone(&payload); + let requests = Arc::clone(&requests); + async move { + if headers.get("x-testkit-token").and_then(|value| value.to_str().ok()) != Some("accepted") { + return StatusCode::UNAUTHORIZED.into_response(); + } + requests.lock().await.push(uri.path().to_owned()); + match uri.path() { + "/index.m3u8" => "#EXTM3U\n#EXTINF:1,\nsegment.ts\n".into_response(), + "/segment.ts" => Response::new(Body::from(payload.as_ref().clone())), + _ => StatusCode::NOT_FOUND.into_response(), + } + } + } + })); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let headers = BTreeMap::from([(String::from("x-testkit-token"), String::from("accepted"))]); + let received = read_segments(&format!("http://{address}/index.m3u8"), &run_id, 17, 1, &headers).await?; + server.abort(); + + assert_eq!(received, 1); + assert_eq!(requests.lock().await.as_slice(), ["/index.m3u8", "/segment.ts"]); + Ok(()) + } +} diff --git a/tools/tuliprox-testkit/src/lib.rs b/tools/tuliprox-testkit/src/lib.rs new file mode 100644 index 000000000..f3adf046f --- /dev/null +++ b/tools/tuliprox-testkit/src/lib.rs @@ -0,0 +1,57 @@ +#![forbid(unsafe_code)] + +pub mod adaptive; +pub mod bootstrap; +pub mod config; +pub mod control; +pub mod custom_video; +pub mod discovery; +pub mod faults; +pub mod frame; +pub mod hls; +pub mod observation; +pub mod oracle; +pub mod origin_events; +pub mod origin_transport; +pub mod policy; +pub mod protocol; +pub mod report; +pub mod scheduler; +pub mod secret; +pub mod shared; +pub mod transport; +pub mod vod; + +use std::process::ExitCode; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TestkitError { + #[error("invalid configuration: {0}")] + Configuration(String), + #[error("protocol error: {0}")] + Protocol(String), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("HTTP error: {0}")] + Http(#[from] reqwest::Error), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunExit { + Passed, + Failed, + Invalid, + Inconclusive, +} + +impl From for ExitCode { + fn from(value: RunExit) -> Self { + match value { + RunExit::Passed => ExitCode::SUCCESS, + RunExit::Failed => ExitCode::from(1), + RunExit::Invalid => ExitCode::from(2), + RunExit::Inconclusive => ExitCode::from(3), + } + } +} diff --git a/tools/tuliprox-testkit/src/main.rs b/tools/tuliprox-testkit/src/main.rs new file mode 100644 index 000000000..bf8a1f268 --- /dev/null +++ b/tools/tuliprox-testkit/src/main.rs @@ -0,0 +1,3232 @@ +use axum::{ + body::Body, + extract::{Extension, Path, Query, State}, + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post, put}, + Json, Router, +}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use bytes::BytesMut; +use clap::{Parser, Subcommand}; +use futures::StreamExt; +use serde::Serialize; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + fmt::Write, + net::SocketAddr, + path::PathBuf, + process::ExitCode, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tokio::{ + net::TcpListener, + sync::{oneshot, watch, Mutex}, +}; +use tuliprox_testkit::{ + bootstrap::{fixture_password, IsolatedFixture}, + config::{ExpectedPlayback, PostReadAction, RuntimeAssertions, Scenario}, + control::{PlaybackRegistry, StartDisposition}, + custom_video::WireMarkerScanner, + discovery::VirtualIdMap, + faults::{FaultSchedule, OriginFault}, + frame::{Frame, FrameDecoder, FrameValidator}, + hls::read_segments, + observation::TuliproxObserver, + oracle::{AdmissionOracle, Decision as AdmissionDecision, Request as AdmissionRequest}, + origin_events::{BodyCloseReason, OriginEvent, OriginEventKind, OriginStats}, + origin_transport::{ + serve_tracked_media_listener, OriginConnectionMeta, OriginLimitMode, OriginPolicy, OriginTracker, + }, + protocol::{ + AgentId, AgentMessage, Command as ControlCommand, CommandId, Envelope, OriginStreamId, PlaybackEvent, + PlaybackId, PlaybackOutcome, RejectionReason, RunId, RunLease, + }, + report::{ReportEvent, RunReport}, + secret::environment, + transport::{controller_websocket, AgentControlConnection, ControllerState, SharedControllerState}, + RunExit, TestkitError, +}; + +fn is_rejection_http_status(status: u16, headers: &reqwest::header::HeaderMap) -> bool { + matches!(status, 403 | 429) || ((status == 502 || status == 503) && headers.get("x-tuliprox-rejection").is_some()) +} + +/// Tuliprox answers a suppressed reentry retry with `204 No Content`: a success status +/// and an empty body. Classify it as an admission rejection so a scenario can assert the +/// quiet termination instead of observing a bare end-of-body. +fn quiet_suppression_outcome(status: u16) -> Option { + (status == 204).then_some(PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(status) }) +} + +/// Build the final playback URL for a live channel, dispatching between M3U-discovered URLs and +/// Xtream live endpoint URLs constructed from the virtual ID embedded in the M3U playlist entry. +fn resolve_playback_url( + endpoint: tuliprox_testkit::config::PlaybackEndpoint, + channel_protocol: &str, + base_url: &str, + username: &str, + password: &str, + vids: &VirtualIdMap, + marker: u32, +) -> Result { + use tuliprox_testkit::config::PlaybackEndpoint; + match endpoint { + PlaybackEndpoint::M3u => Ok(vids.playback_url(marker)?.to_owned()), + PlaybackEndpoint::Xtream => { + if channel_protocol != "xtream_ts" { + return Err(TestkitError::Configuration(format!( + "playback_endpoint: xtream requires protocol xtream_ts, got {channel_protocol}" + ))); + } + let virtual_id = vids.virtual_id_from_marker(marker)?; + let username_enc = url::form_urlencoded::byte_serialize(username.as_bytes()).collect::(); + let password_enc = url::form_urlencoded::byte_serialize(password.as_bytes()).collect::(); + Ok(format!("{base_url}/live/{username_enc}/{password_enc}/{virtual_id}.ts")) + } + } +} + +const PLAYBACK_IDLE_TIMEOUT: Duration = Duration::from_secs(15); +const PLAYLIST_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Parser)] +#[command(name = "tuliprox-testkit", version, about = "Tuliprox streaming admission test kit")] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Serve a controllable synthetic live-stream origin. + Origin { + #[arg(long, default_value = "127.0.0.1:9910")] + listen: SocketAddr, + #[arg(long)] + control_listen: Option, + #[arg(long, default_value = "local-run")] + run_id: String, + #[arg(long, default_value_t = 64_000)] + bitrate: u32, + #[arg(long, value_delimiter = ',', default_value = "17")] + markers: Vec, + #[arg(long)] + account_limit: Option, + #[arg(long, default_value = "observe_only")] + limit_mode: String, + }, + /// Execute a local, sequential scenario against Tuliprox URLs. + Controller { + #[arg(long)] + scenario: PathBuf, + #[arg(long, default_value = "testkit-report")] + report_directory: PathBuf, + #[arg(long, default_value = "127.0.0.1:9900")] + agent_listen: SocketAddr, + #[arg(long)] + run_id: Option, + }, + /// Read and validate one framed playback URL; useful on a remote listener host. + Agent { + #[arg(long)] + url: String, + #[arg(long)] + expected_marker: u32, + #[arg(long, default_value_t = 5)] + frames: u64, + #[arg(long)] + hls: bool, + #[arg(long)] + controller: Option, + #[arg(long, default_value = "local-agent")] + agent_id: String, + #[arg(long, default_value = "local-run")] + run_id: String, + }, +} + +#[derive(Clone)] +struct OriginState { + run_id: RunId, + bitrate: u32, + markers: Arc>, + stream_counter: Arc>, + observations: Arc>>, + faults: Arc>, + hls_sequences: Arc>>, + tracker: OriginTracker, +} + +#[derive(Clone, Serialize)] +struct OriginObservation { + origin_stream_id: String, + channel_marker: u32, + frames_emitted: u64, + bytes_emitted: u64, +} + +#[derive(Serialize)] +struct OriginInstance { + run_id: String, +} + +#[tokio::main] +async fn main() -> ExitCode { + match run(Cli::parse()).await { + Ok(exit) => exit.into(), + Err(error) => { + eprintln!("{error}"); + match error { + TestkitError::Configuration(_) => RunExit::Invalid.into(), + TestkitError::Protocol(_) | TestkitError::Io(_) | TestkitError::Http(_) => RunExit::Inconclusive.into(), + } + } + } +} + +async fn run(cli: Cli) -> Result { + match cli.command { + Command::Origin { listen, control_listen, run_id, bitrate, mut markers, account_limit, limit_mode } => { + markers.sort_unstable(); + markers.dedup(); + if markers.is_empty() { + return Err(TestkitError::Configuration("origin requires at least one channel marker".to_owned())); + } + let mode = match limit_mode.as_str() { + "reject_new" => OriginLimitMode::RejectNew, + "evict_oldest" => OriginLimitMode::EvictOldest, + _ => OriginLimitMode::ObserveOnly, + }; + let policy = OriginPolicy { account_limit, limit_mode: mode }; + let tracker = OriginTracker::new(&run_id, policy); + serve_origin( + listen, + control_listen, + OriginState { + run_id: RunId::new(run_id), + bitrate, + markers: Arc::new(markers), + stream_counter: Arc::new(Mutex::new(0)), + observations: Arc::new(Mutex::new(Vec::new())), + faults: Arc::new(Mutex::new(FaultSchedule::default())), + hls_sequences: Arc::new(Mutex::new(HashMap::new())), + tracker, + }, + ) + .await + } + Command::Agent { url, expected_marker, frames, hls, controller, agent_id, run_id } => { + if let Some(controller) = controller { + return run_controlled_agent(&controller, AgentId::new(agent_id), RunId::new(run_id)).await; + } + if hls { + match read_segments(&url, &RunId::new(run_id), expected_marker, frames, &BTreeMap::new()).await { + Ok(received) => { + println!("received {received} valid HLS frames for marker {expected_marker}"); + Ok(RunExit::Passed) + } + Err(error) => { + eprintln!("{error}"); + Ok(RunExit::Failed) + } + } + } else { + let outcome = run_agent(&url, &RunId::new(run_id), expected_marker, frames, &BTreeMap::new()).await?; + Ok(if outcome.is_streaming() { RunExit::Passed } else { RunExit::Failed }) + } + } + Command::Controller { scenario, report_directory, agent_listen, run_id } => { + run_controller(&scenario, &report_directory, agent_listen, run_id.as_deref()).await + } + } +} + +#[allow(clippy::too_many_lines)] +async fn run_controlled_agent(controller: &str, agent_id: AgentId, run_id: RunId) -> Result { + let hello = Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: "agent-hello".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Hello { + hostname: "remote-agent".to_owned(), + supported_protocols: vec!["xtream_ts".to_owned(), "hls".to_owned()], + maximum_listeners: 1, + }, + }; + let mut connection = AgentControlConnection::connect(controller, hello).await?; + connection + .send_event(Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: "agent-ready".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: 2, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Ready, + }) + .await?; + let mut sequence = 3; + let mut lease = RunLease::new(1, Duration::from_secs(15)); + let mut active: HashMap> = HashMap::new(); + let (term_tx, mut term_rx) = tokio::sync::mpsc::channel::<(PlaybackId, PlaybackOutcome)>(64); + let mut pending_stops: HashMap> = HashMap::new(); + let mut playback_registry = PlaybackRegistry::default(); + let mut heartbeats = tokio::time::interval(Duration::from_secs(5)); + heartbeats.tick().await; + loop { + let command = tokio::select! { + Some((playback_id, outcome)) = term_rx.recv() => { + active.remove(&playback_id.0); + let caused_by = pending_stops.remove(&playback_id.0).flatten(); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: outcome.to_string(), + typed_outcome: Some(outcome), + }, + ) + .await?; + continue; + } + command = connection.next_command() => command?, + _ = heartbeats.tick() => { + connection.send_event(Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: format!("heartbeat-{sequence}"), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: sequence, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Heartbeat, + }).await?; + sequence = sequence.saturating_add(1); + continue; + } + () = tokio::time::sleep_until(lease.deadline()) => { + release_active_playbacks(&mut active); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + None, + PlaybackEvent::LeaseExpired, + ).await?; + return Ok(RunExit::Inconclusive); + } + }; + let Some(command) = command else { + release_active_playbacks(&mut active); + return Ok(RunExit::Inconclusive); + }; + if lease.expired() { + release_active_playbacks(&mut active); + return Ok(RunExit::Inconclusive); + } + if command.run_id != run_id || command.run_generation != 1 { + return Err(TestkitError::Protocol("controller sent a stale or foreign command".to_owned())); + } + let caused_by = command.caused_by_command_id.clone(); + let command_id = caused_by.clone().unwrap_or_else(|| CommandId::new(command.message_id.clone())); + match command.payload { + ControlCommand::StartPlayback { + playback_id, + url, + headers, + expected_run_id, + expected_marker, + required_frames, + } => { + let fingerprint = playback_start_fingerprint(&url, &headers); + match playback_registry.register_start(run_id.clone(), 1, playback_id.clone(), fingerprint)? { + StartDisposition::Replay => { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::CommandAccepted { command_id }, + ) + .await?; + continue; + } + StartDisposition::New => {} + } + let target_run_id = expected_run_id.unwrap_or_else(|| run_id.clone()); + let marker = expected_marker.map_or_else(|| marker_from_url(&url), Ok)?; + let frames = required_frames.unwrap_or(5); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by.clone(), + PlaybackEvent::CommandAccepted { command_id }, + ) + .await?; + match start_held_playback(playback_id.0.clone(), &target_run_id, &url, marker, frames, &headers).await? + { + Ok(held) => { + let term_tx_clone = term_tx.clone(); + let p_id = playback_id.clone(); + tokio::spawn(async move { + let outcome = match held.task.await { + Ok(Ok(outcome)) => outcome, + Ok(Err(err)) => PlaybackOutcome::InfrastructureError { message: err.to_string() }, + Err(join_err) => PlaybackOutcome::InfrastructureError { message: join_err.to_string() }, + }; + let _ = term_tx_clone.send((p_id, outcome)).await; + }); + active.insert(playback_id.0.clone(), held.release); + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::FirstValidFrame { playback_id, sequence: 0 }, + ) + .await?; + } + Err(outcome) => { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: outcome.to_string(), + typed_outcome: Some(outcome), + }, + ) + .await?; + } + } + } + ControlCommand::StopAll => { + release_active_playbacks(&mut active); + return Ok(RunExit::Passed); + } + ControlCommand::StopPlayback { playback_id } => { + if !playback_registry.stop(&run_id, 1, &playback_id) { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: "protocol_rejected".to_owned(), + typed_outcome: Some(PlaybackOutcome::InfrastructureError { + message: "protocol_rejected".to_owned(), + }), + }, + ) + .await?; + continue; + } + if let Some(release) = active.remove(&playback_id.0) { + pending_stops.insert(playback_id.0.clone(), caused_by); + let _ = release.send(()); + } else { + send_agent_event( + &connection, + &run_id, + &agent_id, + &mut sequence, + caused_by, + PlaybackEvent::Terminal { + playback_id, + outcome: "cancelled".to_owned(), + typed_outcome: Some(PlaybackOutcome::ExplicitStop), + }, + ) + .await?; + } + } + ControlCommand::ConfigureRun { lease_millis } | ControlCommand::RenewRunLease { lease_millis } => { + lease.renew(1, Duration::from_millis(lease_millis.max(1)))?; + } + } + } +} + +fn playback_start_fingerprint(url: &str, headers: &BTreeMap) -> String { + let mut source = String::with_capacity(url.len().saturating_add(headers.len().saturating_mul(32))); + source.push_str(url); + for (name, value) in headers { + source.push('\n'); + source.push_str(name); + source.push(':'); + source.push_str(value); + } + blake3::hash(source.as_bytes()).to_hex().to_string() +} + +fn release_active_playbacks(active: &mut HashMap>) { + for (_, release) in active.drain() { + let _ = release.send(()); + } +} + +async fn send_agent_event( + connection: &AgentControlConnection, + run_id: &RunId, + agent_id: &AgentId, + sequence: &mut u64, + caused_by_command_id: Option, + event: PlaybackEvent, +) -> Result<(), TestkitError> { + connection + .send_event(Envelope { + schema_version: 1, + run_id: run_id.clone(), + run_generation: 1, + message_id: format!("event-{sequence}"), + agent_id: agent_id.clone(), + agent_boot_id: "agent-boot".to_owned(), + source_sequence: *sequence, + caused_by_command_id, + local_elapsed_nanos: 0, + payload: AgentMessage::Event { event }, + }) + .await?; + *sequence = sequence.saturating_add(1); + Ok(()) +} + +struct BodyDropGuard { + conn_id: u64, + req_id: u64, + tracker: OriginTracker, + bytes_emitted: Arc, + start_time: Instant, + closed: Arc, + evicted: Arc, +} + +impl Drop for BodyDropGuard { + fn drop(&mut self) { + if !self.closed.swap(true, Ordering::SeqCst) { + let conn_id = self.conn_id; + let req_id = self.req_id; + let tracker = self.tracker.clone(); + let bytes = self.bytes_emitted.load(Ordering::Relaxed); + let duration_ms = u64::try_from(self.start_time.elapsed().as_millis()).unwrap_or(u64::MAX); + let reason = if self.evicted.load(Ordering::Relaxed) { + BodyCloseReason::Evicted + } else { + BodyCloseReason::ClientDisconnected + }; + tokio::spawn(async move { + tracker.on_body_closed(conn_id, req_id, bytes, duration_ms, reason).await; + }); + } + } +} + +async fn serve_origin( + media_listen: SocketAddr, + control_listen: Option, + state: OriginState, +) -> Result { + let media_listener = TcpListener::bind(media_listen).await?; + let media_app = Router::new() + .route("/catalog/input.m3u", get(catalog)) + .route("/live/{*stream}", get(live)) + .route("/hls/{*resource}", get(hls)) + .route("/vod/{*object}", get(vod).head(vod_head)) + .with_state(state.clone()); + + let control_app = Router::new() + .route("/health", get(|| async { StatusCode::NO_CONTENT })) + .route("/v1/instance", get(origin_instance)) + .route("/v1/runs/{run_id}/reset", post(reset_origin_run)) + .route("/v1/runs/{run_id}/connections", get(origin_connections)) + .route("/v1/runs/{run_id}/events", get(origin_events)) + .route("/v1/runs/{run_id}/stats", get(origin_stats)) + .route("/v1/runs/{run_id}/faults/{fault_id}", put(set_fault).delete(clear_fault)) + .with_state(state.clone()); + + let control_addr = control_listen.unwrap_or_else(|| SocketAddr::from(([127, 0, 0, 1], 0))); + let control_listener = TcpListener::bind(control_addr).await?; + let control_task = tokio::spawn(async move { + let _ = axum::serve(control_listener, control_app).await; + }); + + let tracker = state.tracker.clone(); + let media_res = serve_tracked_media_listener(media_listener, media_app, tracker).await; + + control_task.abort(); + media_res.map_err(TestkitError::Io)?; + Ok(RunExit::Passed) +} + +fn catalog_m3u(state: &OriginState, host: &str) -> String { + let mut catalog = String::from("#EXTM3U\n"); + for marker in state.markers.iter() { + let _ = writeln!(catalog, "#EXTINF:-1 tvg-id=\"test-{marker}\",Test channel {marker}"); + let _ = writeln!(catalog, "http://{host}/live/{marker}.ts?run={}", state.run_id.0); + } + let _ = writeln!(catalog, "#EXTINF:-1 tvg-id=\"test-vod-movie.mkv\" tvg-type=\"movie\",Test Movie"); + let _ = writeln!(catalog, "http://{host}/vod/movie.mkv?run={}", state.run_id.0); + catalog +} + +async fn catalog(State(state): State, headers: HeaderMap) -> impl IntoResponse { + let host = headers.get(header::HOST).and_then(|value| value.to_str().ok()).unwrap_or("127.0.0.1"); + ([(header::CONNECTION, "close")], catalog_m3u(&state, host)) +} + +struct LiveStreamState { + ticker: tokio::time::Interval, + evict_rx: oneshot::Receiver<()>, + evicted_flag: Arc, + run_id: RunId, + stream_id: OriginStreamId, + marker: u32, + index: u64, + observations: Arc>>, + observed_stream_id: String, + bytes_emitted: Arc, + _guard: Arc, +} + +#[allow(clippy::too_many_lines)] +async fn live( + Path(stream): Path, + Query(query): Query>, + State(state): State, + meta: Option>, + headers: HeaderMap, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let Ok(marker) = stream.strip_suffix(".ts").unwrap_or(&stream).parse::() else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !state.markers.contains(&marker) { + return StatusCode::NOT_FOUND.into_response(); + } + if matches!(state.faults.lock().await.take("close-before-first-byte"), Some(OriginFault::CloseBeforeFirstByte)) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + + let conn_id = meta.as_ref().map_or(0, |m| m.conn_id); + let range_str = headers.get(header::RANGE).and_then(|v| v.to_str().ok()); + let ua = headers.get(header::USER_AGENT).and_then(|v| v.to_str().ok()); + let req_res = state + .tracker + .on_request_started( + conn_id, + "GET", + &format!("/live/{stream}"), + range_str, + ua, + query.get("account").map(String::as_str), + ) + .await; + + let req_id = match req_res { + Ok(id) => id, + Err((_id, status)) => { + return StatusCode::from_u16(status).unwrap_or(StatusCode::TOO_MANY_REQUESTS).into_response() + } + }; + + let mut counter = state.stream_counter.lock().await; + *counter += 1; + let stream_id = OriginStreamId::new(format!("origin-{}", *counter)); + drop(counter); + state.observations.lock().await.push(OriginObservation { + origin_stream_id: stream_id.0.clone(), + channel_marker: marker, + frames_emitted: 0, + bytes_emitted: 0, + }); + + let evict_rx = state.tracker.on_body_started(conn_id, req_id, None).await; + + let run_id = state.run_id.clone(); + let bitrate = state.bitrate; + let mut sample = BytesMut::new(); + if Frame::synthetic(&run_id, &stream_id, marker, 0).encode(&mut sample).is_err() { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + let bits = u128::try_from(sample.len()).unwrap_or(u128::MAX).saturating_mul(8); + let nanos = bits.saturating_mul(1_000_000_000).checked_div(u128::from(bitrate.max(1))).unwrap_or(1); + let interval = Duration::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX).max(1)); + let observations = state.observations.clone(); + let observed_stream_id = stream_id.0.clone(); + + let bytes_emitted = Arc::new(AtomicU64::new(0)); + let closed = Arc::new(AtomicBool::new(false)); + let evicted = Arc::new(AtomicBool::new(false)); + let guard = Arc::new(BodyDropGuard { + conn_id, + req_id, + tracker: state.tracker.clone(), + bytes_emitted: bytes_emitted.clone(), + start_time: Instant::now(), + closed: closed.clone(), + evicted: evicted.clone(), + }); + + let evicted_flag = evicted.clone(); + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + let stream_state = LiveStreamState { + ticker, + evict_rx, + evicted_flag, + run_id, + stream_id, + marker, + index: 0, + observations, + observed_stream_id, + bytes_emitted, + _guard: guard, + }; + + let body = futures::stream::unfold(stream_state, |mut state| async move { + tokio::select! { + biased; + res = &mut state.evict_rx => { + if res.is_ok() { + state.evicted_flag.store(true, Ordering::Release); + } + None + } + _ = state.ticker.tick() => { + let frame = Frame::synthetic(&state.run_id, &state.stream_id, state.marker, state.index); + state.index = state.index.saturating_add(1); + let mut encoded = BytesMut::new(); + if frame.encode(&mut encoded).is_err() { + return None; + } + let bytes = encoded.len() as u64; + state.bytes_emitted.fetch_add(bytes, Ordering::Relaxed); + if let Some(observation) = state.observations + .lock() + .await + .iter_mut() + .find(|observation| observation.origin_stream_id == state.observed_stream_id) + { + observation.frames_emitted = observation.frames_emitted.saturating_add(1); + observation.bytes_emitted = observation.bytes_emitted.saturating_add(bytes); + } + Some((Ok::<_, std::convert::Infallible>(encoded.freeze()), state)) + } + } + }); + let mut response = Response::new(Body::from_stream(body)); + response.headers_mut().insert(header::CONTENT_TYPE, HeaderValue::from_static("video/mp2t")); + if let Ok(value) = HeaderValue::from_str(&bitrate.to_string()) { + response.headers_mut().insert("x-tpx-test-bitrate", value); + } + response +} + +async fn origin_connections(Path(run_id): Path, State(state): State) -> Response { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND.into_response(); + } + Json(state.observations.lock().await.clone()).into_response() +} + +async fn origin_events(Path(run_id): Path, State(state): State) -> Response { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND.into_response(); + } + Json(state.tracker.events().await).into_response() +} + +async fn origin_stats(Path(run_id): Path, State(state): State) -> Response { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND.into_response(); + } + Json(state.tracker.stats().await).into_response() +} + +async fn origin_instance(State(state): State) -> Json { + Json(OriginInstance { run_id: state.run_id.0.clone() }) +} + +async fn reset_origin_run(Path(run_id): Path, State(state): State) -> StatusCode { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND; + } + *state.stream_counter.lock().await = 0; + state.hls_sequences.lock().await.clear(); + state.observations.lock().await.clear(); + state.tracker.reset().await; + StatusCode::NO_CONTENT +} + +async fn set_fault(Path((run_id, fault_id)): Path<(String, String)>, State(state): State) -> StatusCode { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND; + } + let fault = match fault_id.as_str() { + "close-before-first-byte" => OriginFault::CloseBeforeFirstByte, + _ => return StatusCode::BAD_REQUEST, + }; + match state.faults.lock().await.insert(fault_id, fault) { + Ok(()) => StatusCode::NO_CONTENT, + Err(_) => StatusCode::CONFLICT, + } +} + +async fn clear_fault(Path((run_id, fault_id)): Path<(String, String)>, State(state): State) -> StatusCode { + if run_id != state.run_id.0 { + return StatusCode::NOT_FOUND; + } + match state.faults.lock().await.clear(&fault_id) { + Ok(()) => StatusCode::NO_CONTENT, + Err(_) => StatusCode::NOT_FOUND, + } +} + +async fn hls( + Path(resource): Path, + Query(query): Query>, + State(state): State, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let Some((marker, resource)) = resource.split_once('/') else { + return StatusCode::NOT_FOUND.into_response(); + }; + let Ok(marker) = marker.parse::() else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !state.markers.contains(&marker) { + return StatusCode::NOT_FOUND.into_response(); + } + if resource == "index.m3u8" { + let mut sequences = state.hls_sequences.lock().await; + let sequence = sequences.entry(marker).or_insert(0); + *sequence += 1; + let first = sequence.saturating_sub(1); + let playlist = format!( + "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:1\n#EXT-X-MEDIA-SEQUENCE:{first}\n#EXTINF:1.0,\n{first}.ts?run={}\n#EXTINF:1.0,\n{}.ts?run={}\n", + state.run_id.0, *sequence, state.run_id.0 + ); + return ([(header::CONTENT_TYPE, "application/vnd.apple.mpegurl")], playlist).into_response(); + } + let segment = resource.strip_suffix(".ts").and_then(|part| part.parse::().ok()); + let Some(sequence) = segment else { + return StatusCode::NOT_FOUND.into_response(); + }; + let frame = + Frame::synthetic(&state.run_id, &OriginStreamId::new(format!("hls-presentation-{marker}")), marker, sequence); + let mut encoded = BytesMut::new(); + if frame.encode(&mut encoded).is_err() { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + ([(header::CONTENT_TYPE, "video/mp2t")], encoded.freeze()).into_response() +} + +async fn vod( + Path(object): Path, + Query(query): Query>, + State(state): State, + meta: Option>, + headers: HeaderMap, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let conn_id = meta.as_ref().map_or(0, |m| m.conn_id); + let total_size = + query.get("size").and_then(|s| s.parse::().ok()).unwrap_or(tuliprox_testkit::vod::DEFAULT_VOD_SIZE); + + let range_header = headers.get(header::RANGE).and_then(|value| value.to_str().ok()); + let ua = headers.get(header::USER_AGENT).and_then(|value| value.to_str().ok()); + let req_res = state + .tracker + .on_request_started( + conn_id, + "GET", + &format!("/vod/{object}"), + range_header, + ua, + query.get("account").map(String::as_str), + ) + .await; + + let req_id = match req_res { + Ok(id) => id, + Err((_id, status)) => { + return StatusCode::from_u16(status).unwrap_or(StatusCode::TOO_MANY_REQUESTS).into_response() + } + }; + + let (start, end, status) = match range_header { + None => (0, total_size, StatusCode::OK), + Some(range_str) => { + if let Some((start, end)) = tuliprox_testkit::vod::parse_range_header(range_str, total_size) { + (start, end, StatusCode::PARTIAL_CONTENT) + } else { + let mut response = StatusCode::RANGE_NOT_SATISFIABLE.into_response(); + if let Ok(value) = HeaderValue::from_str(&format!("bytes */{total_size}")) { + response.headers_mut().insert(header::CONTENT_RANGE, value); + } + apply_vod_validators(&mut response, &vod_payload(&object)); + return response; + } + } + }; + + let length = end.saturating_sub(start); + let evict_rx = state.tracker.on_body_started(conn_id, req_id, Some(length)).await; + let stall_ms = query.get("stall_ms").and_then(|s| s.parse::().ok()); + let abort_after_bytes = query.get("abort_after_bytes").and_then(|s| s.parse::().ok()); + + let options = tuliprox_testkit::vod::VodStreamOptions { + object: object.clone(), + start_offset: start, + length, + bitrate: state.bitrate, + chunk_size: 64 * 1024, + stall_ms, + abort_after_bytes, + }; + + let bytes_emitted = Arc::new(AtomicU64::new(0)); + let closed = Arc::new(AtomicBool::new(false)); + let evicted = Arc::new(AtomicBool::new(false)); + let guard = Arc::new(BodyDropGuard { + conn_id, + req_id, + tracker: state.tracker.clone(), + bytes_emitted: bytes_emitted.clone(), + start_time: Instant::now(), + closed: closed.clone(), + evicted: evicted.clone(), + }); + let body_stream = tuliprox_testkit::vod::create_vod_stream_with_evicted(options, evict_rx, Some(evicted)); + + let bytes_tracker = bytes_emitted.clone(); + let tracked_stream = body_stream.map(move |item| { + let _keep_guard = &guard; + if let Ok(chunk) = &item { + bytes_tracker.fetch_add(chunk.len() as u64, Ordering::Relaxed); + } + item + }); + + let mut response = Response::new(Body::from_stream(tracked_stream)); + *response.status_mut() = status; + response.headers_mut().insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + if status == StatusCode::PARTIAL_CONTENT { + let value = format!("bytes {}-{}/{}", start, end.saturating_sub(1), total_size); + if let Ok(value) = HeaderValue::from_str(&value) { + response.headers_mut().insert(header::CONTENT_RANGE, value); + } + } + if let Ok(value) = HeaderValue::from_str(&length.to_string()) { + response.headers_mut().insert(header::CONTENT_LENGTH, value); + } + apply_vod_validators(&mut response, &vod_payload(&object)); + response +} + +async fn vod_head( + Path(object): Path, + Query(query): Query>, + State(state): State, +) -> Response { + if query.get("run") != Some(&state.run_id.0) { + return StatusCode::NOT_FOUND.into_response(); + } + let total_size = + query.get("size").and_then(|s| s.parse::().ok()).unwrap_or(tuliprox_testkit::vod::DEFAULT_VOD_SIZE); + let mut response = StatusCode::OK.into_response(); + response.headers_mut().insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + if let Ok(value) = HeaderValue::from_str(&total_size.to_string()) { + response.headers_mut().insert(header::CONTENT_LENGTH, value); + } + apply_vod_validators(&mut response, &vod_payload(&object)); + response +} + +fn vod_payload(object: &str) -> Vec { + let digest = blake3::hash(object.as_bytes()); + digest.as_bytes().iter().copied().cycle().take(16 * 1024).collect() +} + +fn apply_vod_validators(response: &mut Response, payload: &[u8]) { + let etag = format!("\"{}\"", blake3::hash(payload).to_hex()); + if let Ok(etag) = HeaderValue::from_str(&etag) { + response.headers_mut().insert(header::ETAG, etag); + } + response.headers_mut().insert(header::LAST_MODIFIED, HeaderValue::from_static("Thu, 01 Jan 1970 00:00:00 GMT")); +} + +async fn run_agent( + url: &str, + expected_run_id: &RunId, + expected_marker: u32, + required_frames: u64, + headers: &BTreeMap, +) -> Result { + let client = reqwest::Client::builder().http1_only().pool_max_idle_per_host(0).build()?; + let mut request = client.get(url); + for (name, value) in headers { + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header name {name}: {error}")) + })?; + let value = reqwest::header::HeaderValue::from_str(value).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header value for {name}: {error}")) + })?; + request = request.header(name, value); + } + let response = match request.send().await { + Ok(res) => res, + Err(err) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + }; + let status = response.status().as_u16(); + if !response.status().is_success() { + eprintln!("playback returned {status}"); + return Ok(if is_rejection_http_status(status, response.headers()) { + PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(status) } + } else { + PlaybackOutcome::HttpError { status } + }); + } + if let Some(outcome) = quiet_suppression_outcome(status) { + return Ok(outcome); + } + let mut body = response.bytes_stream(); + let mut decoder = FrameDecoder::default(); + let mut validator = FrameValidator::new(expected_run_id, expected_marker); + let mut custom_video = WireMarkerScanner::default(); + let mut received_frames = 0; + let mut received_bytes = 0; + loop { + let chunk = match tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()).await { + Ok(Some(Ok(chunk))) => chunk, + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => break, + Err(_) => { + eprintln!("stream did not produce bytes before the idle deadline"); + return Ok(PlaybackOutcome::IdleTimeout); + } + }; + received_bytes += chunk.len() as u64; + if let Some(kind) = custom_video.push(&chunk)? { + eprintln!("received custom-video fallback: {kind:?}"); + if matches!( + kind, + tuliprox_testkit::custom_video::CustomVideoKind::UserConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::ProviderConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::LowPriorityPreempted + | tuliprox_testkit::custom_video::CustomVideoKind::UserAccountExpired + ) { + return Ok(PlaybackOutcome::AdmissionRejected { reason: RejectionReason::CustomVideo(kind) }); + } + return Ok(PlaybackOutcome::InvalidData { + message: format!("unexpected non-admission custom video: {kind:?}"), + }); + } + for frame in decoder.push(&chunk)? { + if let Err(error) = validator.validate(&frame) { + eprintln!("{error}"); + return Ok(PlaybackOutcome::InvalidData { message: error.to_string() }); + } + received_frames += 1; + if received_frames >= required_frames { + println!("received {received_frames} valid frames for marker {expected_marker}"); + return Ok(PlaybackOutcome::Streaming { frames: received_frames, bytes: received_bytes }); + } + } + } + eprintln!("stream ended after {received_frames} valid frames"); + Ok(PlaybackOutcome::UnexpectedEof { frames: received_frames, bytes: received_bytes }) +} + +/// Keep the HTTP body consumed after proving its frame identity. Retaining the +/// request alone is not sufficient: a bounded proxy buffer could otherwise make +/// the upstream session disappear before the following admission request. +async fn run_agent_until_released( + url: String, + expected_run_id: RunId, + expected_marker: u32, + required_frames: u64, + headers: BTreeMap, + ready: oneshot::Sender<()>, + mut release: oneshot::Receiver<()>, +) -> Result { + let client = reqwest::Client::builder().http1_only().pool_max_idle_per_host(0).build()?; + let mut request = client.get(url); + for (name, value) in &headers { + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header name {name}: {error}")) + })?; + let value = reqwest::header::HeaderValue::from_str(value).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header value for {name}: {error}")) + })?; + request = request.header(name, value); + } + let response = match request.send().await { + Ok(res) => res, + Err(err) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + }; + let status = response.status().as_u16(); + if !response.status().is_success() { + return Ok(if is_rejection_http_status(status, response.headers()) { + PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(status) } + } else { + PlaybackOutcome::HttpError { status } + }); + } + if let Some(outcome) = quiet_suppression_outcome(status) { + return Ok(outcome); + } + let mut body = response.bytes_stream(); + let mut decoder = FrameDecoder::default(); + let mut validator = FrameValidator::new(&expected_run_id, expected_marker); + let mut custom_video = WireMarkerScanner::default(); + let mut received_frames = 0; + let mut received_bytes = 0; + let mut ready = Some(ready); + loop { + tokio::select! { + _ = &mut release => { + return Ok(if ready.is_none() { + PlaybackOutcome::ExplicitStop + } else { + PlaybackOutcome::UnexpectedEof { frames: received_frames, bytes: received_bytes } + }); + } + chunk = tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()) => { + let chunk = match chunk { + Ok(Some(Ok(chunk))) => chunk, + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => return Ok(PlaybackOutcome::UnexpectedEof { frames: received_frames, bytes: received_bytes }), + Err(_) => return Ok(PlaybackOutcome::IdleTimeout), + }; + received_bytes += chunk.len() as u64; + if let Some(kind) = custom_video.push(&chunk)? { + if matches!( + kind, + tuliprox_testkit::custom_video::CustomVideoKind::UserConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::ProviderConnectionsExhausted + | tuliprox_testkit::custom_video::CustomVideoKind::LowPriorityPreempted + | tuliprox_testkit::custom_video::CustomVideoKind::UserAccountExpired + ) { + return Ok(PlaybackOutcome::AdmissionRejected { + reason: RejectionReason::CustomVideo(kind), + }); + } + return Ok(PlaybackOutcome::InvalidData { + message: format!("unexpected non-admission custom video: {kind:?}"), + }); + } + for frame in decoder.push(&chunk)? { + if let Err(error) = validator.validate(&frame) { + return Ok(PlaybackOutcome::InvalidData { message: error.to_string() }); + } + received_frames += 1; + if received_frames >= required_frames { + if let Some(ready) = ready.take() { + let _ = ready.send(()); + } + } + } + } + } + } +} + +async fn run_hls_until_released( + url: String, + expected_run_id: RunId, + expected_marker: u32, + required_frames: u64, + headers: BTreeMap, + ready: oneshot::Sender<()>, + mut release: oneshot::Receiver<()>, +) -> Result { + let mut ready = Some(ready); + loop { + tokio::select! { + _ = &mut release => { + return Ok(if ready.is_none() { + PlaybackOutcome::ExplicitStop + } else { + PlaybackOutcome::UnexpectedEof { frames: 0, bytes: 0 } + }); + } + result = read_segments(&url, &expected_run_id, expected_marker, required_frames, &headers) => { + if let Err(err) = result { + return Ok(PlaybackOutcome::TransportError { message: err.to_string() }); + } + if let Some(ready) = ready.take() { + let _ = ready.send(()); + } + } + } + tokio::select! { + _ = &mut release => return Ok(PlaybackOutcome::ExplicitStop), + () = tokio::time::sleep(Duration::from_millis(100)) => {} + } + } +} + +fn is_hls_manifest_url(url: &str) -> bool { + url::Url::parse(url).is_ok_and(|parsed| { + std::path::Path::new(parsed.path()).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("m3u8")) + }) +} + +struct HeldPlayback { + playback_id: String, + release: oneshot::Sender<()>, + task: tokio::task::JoinHandle>, +} + +#[derive(Clone)] +pub struct OriginObserver { + control_base_url: String, + client: reqwest::Client, +} + +impl OriginObserver { + #[must_use] + pub fn new(control_base_url: &str) -> Self { + Self { control_base_url: control_base_url.trim_end_matches('/').to_owned(), client: reqwest::Client::new() } + } + + pub async fn stats(&self, run_id: &str) -> Result { + let url = format!("{}/v1/runs/{run_id}/stats", self.control_base_url); + let resp = self.client.get(&url).send().await?; + if !resp.status().is_success() { + return Err(TestkitError::Protocol(format!("origin stats error: {}", resp.status()))); + } + let stats = resp.json::().await?; + Ok(stats) + } + + pub async fn events(&self, run_id: &str) -> Result, TestkitError> { + let url = format!("{}/v1/runs/{run_id}/events", self.control_base_url); + let resp = self.client.get(&url).send().await?; + if !resp.status().is_success() { + return Err(TestkitError::Protocol(format!("origin events error: {}", resp.status()))); + } + let events = resp.json::>().await?; + Ok(events) + } + + pub async fn reset(&self, run_id: &str) -> Result<(), TestkitError> { + let url = format!("{}/v1/runs/{run_id}/reset", self.control_base_url); + let resp = self.client.post(&url).send().await?; + if !resp.status().is_success() { + return Err(TestkitError::Protocol(format!("origin reset error: {}", resp.status()))); + } + Ok(()) + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn run_vod_until_released( + url: String, + vod_object: Option, + method: Option, + range: Option, + expected_status: Option, + expected_content_range: Option, + read_limit_bytes: Option, + post_read_action: PostReadAction, + headers: BTreeMap, + ready: oneshot::Sender<()>, + mut release: oneshot::Receiver<()>, +) -> Result { + let client = reqwest::Client::builder().http1_only().pool_max_idle_per_host(0).build()?; + let method = match method.as_deref() { + Some("HEAD") => reqwest::Method::HEAD, + _ => reqwest::Method::GET, + }; + let mut request = client.request(method.clone(), &url); + for (name, value) in &headers { + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header name {name}: {error}")) + })?; + let value = reqwest::header::HeaderValue::from_str(value).map_err(|error| { + TestkitError::Configuration(format!("invalid playback request header value for {name}: {error}")) + })?; + request = request.header(name, value); + } + if let Some(ref range_str) = range { + let name = reqwest::header::RANGE; + let value = reqwest::header::HeaderValue::from_str(range_str) + .map_err(|error| TestkitError::Configuration(format!("invalid range header value {range_str}: {error}")))?; + request = request.header(name, value); + } + let response = match request.send().await { + Ok(res) => res, + Err(err) => { + return Ok(PlaybackOutcome::TransportError { message: err.to_string() }); + } + }; + let status = response.status().as_u16(); + if is_rejection_http_status(status, response.headers()) { + return Ok(PlaybackOutcome::AdmissionRejected { reason: RejectionReason::HttpStatus(status) }); + } + let expected = expected_status.unwrap_or(if range.is_some() { 206 } else { 200 }); + if status != expected { + return Ok(PlaybackOutcome::HttpError { status }); + } + let start_offset = if range.is_some() || status == 206 { + let Some(cr_header) = response.headers().get(reqwest::header::CONTENT_RANGE).and_then(|v| v.to_str().ok()) + else { + return Ok(PlaybackOutcome::InvalidData { message: "range response omits Content-Range".to_owned() }); + }; + let Some((cr_start, cr_end, cr_total)) = tuliprox_testkit::vod::parse_content_range(cr_header) else { + return Ok(PlaybackOutcome::InvalidData { message: format!("invalid Content-Range header: {cr_header}") }); + }; + if cr_start > cr_end { + return Ok(PlaybackOutcome::InvalidData { + message: format!("Content-Range start exceeds end: {cr_header}"), + }); + } + if let Some(ref exp_cr) = expected_content_range { + if cr_header != exp_cr { + return Ok(PlaybackOutcome::InvalidData { + message: format!("unexpected content-range: {cr_header:?}, expected: {exp_cr}"), + }); + } + } + if let Some(ref r) = range { + let total_for_range = cr_total.unwrap_or(tuliprox_testkit::vod::DEFAULT_VOD_SIZE); + let Some((req_start, req_end_exclusive)) = tuliprox_testkit::vod::parse_range_header(r, total_for_range) + else { + return Ok(PlaybackOutcome::InvalidData { + message: format!("failed to parse requested range header against size {total_for_range}: {r}"), + }); + }; + let req_end_inclusive = req_end_exclusive.saturating_sub(1); + if cr_start != req_start || cr_end != req_end_inclusive { + return Ok(PlaybackOutcome::InvalidData { + message: format!( + "Content-Range {cr_header} does not match requested range {req_start}-{req_end_inclusive}" + ), + }); + } + if let Some(total) = cr_total { + if cr_end >= total { + return Ok(PlaybackOutcome::InvalidData { + message: format!("Content-Range end {cr_end} exceeds or equals total size {total}"), + }); + } + } + } + let expected_len = cr_end.saturating_sub(cr_start).saturating_add(1); + if let Some(cl) = response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + { + if cl != expected_len { + return Ok(PlaybackOutcome::InvalidData { + message: format!("Content-Length {cl} does not match Content-Range length {expected_len}"), + }); + } + } + cr_start + } else { + 0 + }; + if method == reqwest::Method::HEAD { + let _ = ready.send(()); + let _ = release.await; + return Ok(PlaybackOutcome::ExplicitStop); + } + + let object_name = vod_object.as_deref().or_else(|| url.rsplit('/').next()).unwrap_or("movie.mkv"); + + let mut body = response.bytes_stream(); + let mut received_bytes: u64 = 0; + let limit = read_limit_bytes.unwrap_or(u64::MAX); + let mut ready = Some(ready); + + while received_bytes < limit { + tokio::select! { + _ = &mut release => { + return Ok(if ready.is_none() { + PlaybackOutcome::ExplicitStop + } else { + PlaybackOutcome::UnexpectedEof { frames: 0, bytes: received_bytes } + }); + } + chunk = tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()) => { + match chunk { + Ok(Some(Ok(bytes))) => { + let to_take = usize::try_from(limit.saturating_sub(received_bytes)) + .unwrap_or(usize::MAX) + .min(bytes.len()); + let chunk_offset = start_offset.saturating_add(received_bytes); + if !tuliprox_testkit::vod::verify_deterministic_slice(object_name, chunk_offset, &bytes[..to_take]) { + return Ok(PlaybackOutcome::InvalidData { + message: format!("deterministic slice verification failed for {object_name} at offset {chunk_offset}"), + }); + } + received_bytes += to_take as u64; + if received_bytes >= limit || (read_limit_bytes.is_none() && to_take == bytes.len()) { + if let Some(r) = ready.take() { + let _ = r.send(()); + } + } + } + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => { + if let Some(r) = ready.take() { + let _ = r.send(()); + } + return Ok(PlaybackOutcome::UnexpectedEof { frames: 0, bytes: received_bytes }); + } + Err(_) => return Ok(PlaybackOutcome::IdleTimeout), + } + } + } + } + + if let Some(r) = ready.take() { + let _ = r.send(()); + } + + match post_read_action { + PostReadAction::Close => { + drop(body); + Ok(PlaybackOutcome::Streaming { frames: 0, bytes: received_bytes }) + } + PostReadAction::Pause => { + let _ = release.await; + Ok(PlaybackOutcome::ExplicitStop) + } + PostReadAction::KeepOpen => loop { + tokio::select! { + _ = &mut release => return Ok(PlaybackOutcome::ExplicitStop), + chunk = tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, body.next()) => { + match chunk { + Ok(Some(Ok(bytes))) => { + let _ = bytes; + } + Ok(Some(Err(err))) => return Ok(PlaybackOutcome::TransportError { message: err.to_string() }), + Ok(None) => return Ok(PlaybackOutcome::ExplicitStop), + Err(_) => return Ok(PlaybackOutcome::IdleTimeout), + } + } + } + }, + } +} + +#[allow(clippy::too_many_arguments)] +async fn start_held_vod_playback( + playback_id: String, + url: &str, + vod_object: Option, + method: Option, + range: Option, + expected_status: Option, + expected_content_range: Option, + read_limit_bytes: Option, + post_read_action: PostReadAction, + headers: &BTreeMap, +) -> Result, TestkitError> { + let (ready_tx, ready_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let is_close = post_read_action == PostReadAction::Close; + let task = tokio::spawn(run_vod_until_released( + url.to_owned(), + vod_object, + method, + range, + expected_status, + expected_content_range, + read_limit_bytes, + post_read_action, + headers.clone(), + ready_tx, + release_rx, + )); + + match tokio::time::timeout(Duration::from_secs(10), ready_rx).await { + Ok(Ok(())) => { + if is_close { + match task.await { + Ok(Ok(outcome)) => Ok(Err(outcome)), + Ok(Err(err)) => Err(err), + Err(err) => Err(TestkitError::Protocol(format!("task error: {err}"))), + } + } else { + Ok(Ok(HeldPlayback { playback_id, release: release_tx, task })) + } + } + Ok(Err(_)) => match task.await { + Ok(Ok(outcome)) => Ok(Err(outcome)), + Ok(Err(error)) => Err(error), + Err(error) => Err(TestkitError::Protocol(format!("playback task failed: {error}"))), + }, + Err(_) => { + task.abort(); + let _ = task.await; + Ok(Err(PlaybackOutcome::IdleTimeout)) + } + } +} + +async fn start_held_playback( + playback_id: String, + expected_run_id: &RunId, + url: &str, + marker: u32, + frames: u64, + headers: &BTreeMap, +) -> Result, TestkitError> { + let (ready_tx, ready_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let task = if is_hls_manifest_url(url) { + tokio::spawn(run_hls_until_released( + url.to_owned(), + expected_run_id.clone(), + marker, + frames, + headers.clone(), + ready_tx, + release_rx, + )) + } else { + tokio::spawn(run_agent_until_released( + url.to_owned(), + expected_run_id.clone(), + marker, + frames, + headers.clone(), + ready_tx, + release_rx, + )) + }; + match tokio::time::timeout(Duration::from_secs(10), ready_rx).await { + Ok(Ok(())) => Ok(Ok(HeldPlayback { playback_id, release: release_tx, task })), + Ok(Err(_)) => match task.await { + Ok(Ok(outcome)) => Ok(Err(outcome)), + Ok(Err(error)) => Err(error), + Err(error) => Err(TestkitError::Protocol(format!("playback task failed: {error}"))), + }, + Err(_) => { + task.abort(); + let _ = task.await; + Ok(Err(PlaybackOutcome::IdleTimeout)) + } + } +} + +#[allow(clippy::too_many_lines)] +async fn run_controller( + path: &std::path::Path, + report_directory: &std::path::Path, + agent_listen: SocketAddr, + run_id: Option<&str>, +) -> Result { + let scenario_name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown").to_owned(); + let mut scenario = match Scenario::from_path(path) { + Ok(scenario) => scenario, + Err(err) => { + let _ = write_raw_report( + &scenario_name, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + let working_directory = path.parent().unwrap_or_else(|| std::path::Path::new(".")); + let expanded_steps = match scenario.expanded_steps() { + Ok(steps) => steps, + Err(err) => { + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + let fixture = match scenario.tuliprox.execution_mode { + tuliprox_testkit::config::ExecutionMode::IsolatedFixture => { + let Some(bootstrap) = scenario.tuliprox.bootstrap.as_ref() else { + let err = TestkitError::Configuration("isolated_fixture requires a bootstrap definition".to_owned()); + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + }; + let testkit_binary = match std::env::current_exe() { + Ok(b) => b, + Err(err) => { + let err = TestkitError::Io(err); + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("bootstrap_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + let fixture = match IsolatedFixture::start( + bootstrap, + working_directory, + &testkit_binary, + &scenario.name, + scenario.origin.as_ref(), + scenario.policy_contract.as_ref(), + &scenario.channels, + &scenario.tuliprox.fixture_stream, + scenario.tuliprox.playback_endpoint == tuliprox_testkit::config::PlaybackEndpoint::Xtream, + ) + .await + { + Ok(f) => f, + Err(err) => { + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("bootstrap_error"), + Some(&err.to_string()), + Some(4), + ); + return Err(err); + } + }; + scenario.tuliprox.base_url.clone_from(&fixture.base_url); + scenario.tuliprox.api_base_url = Some(format!("{}/api/v1", fixture.base_url)); + let has_vod = + expanded_steps.iter().any(|s| s.start.as_ref().and_then(|st| st.vod_object.as_ref()).is_some()); + if !scenario.channels.is_empty() || has_vod { + let Some(username) = scenario.actors.iter().find_map(|actor| actor.username.as_deref()) else { + let err = + TestkitError::Configuration("channel or vod scenarios require an actor username".to_owned()); + let _ = write_report( + &scenario, + report_directory, + RunExit::Failed, + Vec::new(), + "pre-start", + run_id, + Some("configuration_error"), + Some(&err.to_string()), + Some(4), + ); + let _ = fixture.stop().await; + return Err(err); + }; + scenario.tuliprox.playlist_url = Some(format!( + "{}/m3u?username={}&password={}", + fixture.base_url, + url::form_urlencoded::byte_serialize(username.as_bytes()).collect::(), + fixture_password(), + )); + } + Some(fixture) + } + tuliprox_testkit::config::ExecutionMode::ExistingInstance => None, + }; + let origin_control_url = fixture.as_ref().map(|f| f.origin_control_url.clone()); + let scenario_name_for_err = scenario.name.clone(); + let result = + run_controller_scenario(scenario, report_directory, agent_listen, origin_control_url.as_deref(), run_id).await; + let cleanup = match fixture { + Some(fixture) => fixture.stop().await, + None => Ok(()), + }; + match (result, cleanup) { + (Ok(outcome), Ok(())) => Ok(outcome), + (Ok(_), Err(error)) => { + let _ = write_raw_report( + &scenario_name_for_err, + report_directory, + RunExit::Failed, + Vec::new(), + "cleanup", + run_id, + Some("cleanup_error"), + Some(&error.to_string()), + Some(4), + ); + Err(error) + } + (Err(error), _) => { + let _ = write_raw_report( + &scenario_name_for_err, + report_directory, + RunExit::Failed, + Vec::new(), + "execution_error", + run_id, + Some("execution_error"), + Some(&error.to_string()), + Some(4), + ); + Err(error) + } + } +} + +#[allow(clippy::too_many_lines)] +async fn run_controller_scenario( + scenario: Scenario, + report_directory: &std::path::Path, + agent_listen: SocketAddr, + origin_control_url: Option<&str>, + run_id: Option<&str>, +) -> Result { + if scenario.tuliprox.execution_mode == tuliprox_testkit::config::ExecutionMode::ExistingInstance { + eprintln!("existing-instance mode is intentionally inconclusive until loaded-policy evidence is available"); + write_report( + &scenario, + report_directory, + RunExit::Inconclusive, + Vec::new(), + "unproven-existing-instance-policy", + run_id, + Some("execution_mode_unsupported"), + Some("existing-instance mode is intentionally inconclusive until loaded-policy evidence is available"), + Some(3), + )?; + return Ok(RunExit::Inconclusive); + } + let api_base_url = scenario + .tuliprox + .api_base_url + .clone() + .unwrap_or_else(|| format!("{}/api/v1", scenario.tuliprox.base_url.trim_end_matches('/'))); + let credentials = match (&scenario.tuliprox.admin_username, &scenario.tuliprox.admin_password_env) { + (Some(username), Some(password_env)) => Some((username.clone(), environment(password_env)?)), + (None, None) => None, + _ => { + return Err(TestkitError::Configuration( + "admin_username and admin_password_env must be specified together".to_owned(), + )) + } + }; + let observer = TuliproxObserver::new(api_base_url, credentials); + let policy_hash = match observer.config().await { + Ok(config) => { + if let Some(contract) = &scenario.policy_contract { + match tuliprox_testkit::policy::validate_fixture_policy(contract, &config) { + Ok(()) => {} + Err(TestkitError::Configuration(error)) => { + eprintln!("fixture policy contract rejected: {error}"); + write_report( + &scenario, + report_directory, + RunExit::Invalid, + Vec::new(), + "policy-mismatch", + run_id, + Some("policy_mismatch"), + Some(&error), + Some(2), + )?; + return Ok(RunExit::Invalid); + } + Err(error) => { + eprintln!("fixture policy evidence unavailable: {error}"); + write_report( + &scenario, + report_directory, + RunExit::Inconclusive, + Vec::new(), + "policy-unreadable", + run_id, + Some("policy_unreadable"), + Some(&error.to_string()), + Some(3), + )?; + return Ok(RunExit::Inconclusive); + } + } + } + blake3::hash(config.to_string().as_bytes()).to_hex().to_string() + } + Err(error) => { + eprintln!("configuration observation unavailable: {error}"); + write_report( + &scenario, + report_directory, + RunExit::Inconclusive, + Vec::new(), + "unavailable", + run_id, + Some("configuration_unavailable"), + Some(&error.to_string()), + Some(3), + )?; + return Ok(RunExit::Inconclusive); + } + }; + let expanded_steps = scenario.expanded_steps()?; + let requires_discovery = expanded_steps.iter().any(|step| { + step.start.as_ref().and_then(|start| start.channel.as_ref().or(start.vod_object.as_ref())).is_some() + }); + let virtual_ids_by_user = if requires_discovery { + let required_markers = channel_markers(&scenario, &expanded_steps)?; + let required_vod = vod_objects(&expanded_steps); + let mut map = HashMap::new(); + let usernames: std::collections::BTreeSet = + scenario.actors.iter().filter_map(|actor| actor.username.clone()).collect(); + if usernames.is_empty() { + let playlist_url = scenario.tuliprox.playlist_url.as_deref().ok_or_else(|| { + TestkitError::Configuration( + "channel-based or vod-based starts require tuliprox.playlist_url".to_owned(), + ) + })?; + let vids = discover_virtual_ids(playlist_url, &required_markers, &required_vod).await?; + map.insert(String::new(), vids); + } else { + for user in usernames { + let playlist_url = format!( + "{}/m3u?username={}&password={}", + scenario.tuliprox.base_url.trim_end_matches('/'), + url::form_urlencoded::byte_serialize(user.as_bytes()).collect::(), + fixture_password(), + ); + let vids = discover_virtual_ids(&playlist_url, &required_markers, &required_vod).await?; + map.insert(user, vids); + } + } + map + } else { + HashMap::new() + }; + let origin_observer = origin_control_url.map(OriginObserver::new); + let controller_state = start_controller_transport(agent_listen, &scenario.name).await?; + wait_for_required_agents(&controller_state, &scenario.agents.required).await?; + let (stop_lease_renewals, lease_stop) = watch::channel(false); + let lease_renewal_task = + start_agent_lease_renewals(controller_state.clone(), &scenario.agents.required, lease_stop); + let execution_result = execute_scenario_steps( + &scenario, + &expanded_steps, + &observer, + origin_observer.as_ref(), + &controller_state, + &virtual_ids_by_user, + ) + .await; + let _ = stop_lease_renewals.send(true); + let lease_result = lease_renewal_task + .await + .map_err(|error| TestkitError::Protocol(format!("agent lease renewal task failed: {error}")))?; + let (any_failed, observations_incomplete, events) = execution_result?; + lease_result?; + let (outcome, exit_code, error_kind, error_message) = if any_failed { + (RunExit::Failed, Some(1), Some("assertion_failed"), Some("one or more scenario steps failed")) + } else if observations_incomplete { + (RunExit::Inconclusive, Some(3), Some("incomplete_observations"), Some("scenario observations were incomplete")) + } else { + (RunExit::Passed, Some(0), None, None) + }; + write_report( + &scenario, + report_directory, + outcome, + events, + &policy_hash, + run_id, + error_kind, + error_message, + exit_code, + )?; + Ok(outcome) +} + +fn channel_markers(scenario: &Scenario, steps: &[tuliprox_testkit::config::Step]) -> Result, TestkitError> { + let mut markers = + steps + .iter() + .filter_map(|step| step.start.as_ref()) + .filter_map(|start| start.channel.as_deref()) + .map(|channel| { + scenario.channels.get(channel).map(|definition| definition.origin_marker).ok_or_else(|| { + TestkitError::Configuration(format!("channel {channel} is not defined by the scenario")) + }) + }) + .collect::, _>>()?; + markers.sort_unstable(); + markers.dedup(); + Ok(markers) +} + +fn vod_objects(steps: &[tuliprox_testkit::config::Step]) -> Vec { + let mut vods = steps + .iter() + .filter_map(|step| step.start.as_ref()) + .filter_map(|start| start.vod_object.as_deref()) + .map(str::to_owned) + .collect::>(); + vods.sort_unstable(); + vods.dedup(); + vods +} + +async fn discover_virtual_ids( + playlist_url: &str, + required_markers: &[u32], + required_vod_objects: &[String], +) -> Result { + let deadline = tokio::time::Instant::now() + PLAYLIST_DISCOVERY_TIMEOUT; + let client = reqwest::Client::new(); + loop { + let last_reason = match client.get(playlist_url).send().await { + Ok(response) => match response.error_for_status() { + Ok(response) => match response.text().await { + Ok(playlist) => match VirtualIdMap::from_m3u(&playlist) { + Ok(virtual_ids) => { + let missing = required_markers + .iter() + .copied() + .filter(|marker| virtual_ids.playback_url(*marker).is_err()) + .collect::>(); + let missing_vod = required_vod_objects + .iter() + .filter(|name| virtual_ids.named_playback_url(name).is_err()) + .collect::>(); + if missing.is_empty() && missing_vod.is_empty() { + return Ok(virtual_ids); + } + format!("playlist has not published markers {missing:?} or vod {missing_vod:?}") + } + Err(error) => error.to_string(), + }, + Err(error) => error.to_string(), + }, + Err(error) => error.to_string(), + }, + Err(error) => error.to_string(), + }; + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!( + "Tuliprox did not publish the required virtual IDs within {} seconds: {last_reason}", + PLAYLIST_DISCOVERY_TIMEOUT.as_secs() + ))); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +fn start_agent_lease_renewals( + controller_state: ControllerState, + agent_ids: &[String], + mut stop: watch::Receiver, +) -> tokio::task::JoinHandle> { + const LEASE_DURATION: Duration = Duration::from_secs(15); + let agents = agent_ids.iter().cloned().map(AgentId::new).collect::>(); + tokio::spawn(async move { + for agent in &agents { + dispatch_run_lease(&controller_state, agent, LEASE_DURATION, true).await?; + } + let mut interval = tokio::time::interval(LEASE_DURATION / 3); + interval.tick().await; + loop { + tokio::select! { + changed = stop.changed() => { + if changed.is_err() || *stop.borrow() { + return Ok(()); + } + } + _ = interval.tick() => { + for agent in &agents { + dispatch_run_lease(&controller_state, agent, LEASE_DURATION, false).await?; + } + } + } + } + }) +} + +async fn dispatch_run_lease( + controller_state: &ControllerState, + agent: &AgentId, + lease_duration: Duration, + initial: bool, +) -> Result<(), TestkitError> { + let sequence = controller_state.next_command_sequence(); + let command_id = format!("lease-{}-{sequence}", agent.0); + let lease_millis = u64::try_from(lease_duration.as_millis()).unwrap_or(u64::MAX); + controller_state + .dispatch( + agent, + Envelope { + schema_version: 1, + run_id: controller_state.run_id(), + run_generation: 1, + message_id: command_id.clone(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: sequence, + caused_by_command_id: Some(CommandId::new(command_id)), + local_elapsed_nanos: 0, + payload: if initial { + ControlCommand::ConfigureRun { lease_millis } + } else { + ControlCommand::RenewRunLease { lease_millis } + }, + }, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn check_origin_assertions<'a>( + step_id: &'a str, + assert_origin: &tuliprox_testkit::config::AssertOrigin, + obs: &OriginObserver, + origin_run_id: &str, + events: &mut Vec<(&'a str, &'static str)>, + any_failed: &mut bool, + observations_incomplete: &mut bool, +) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + loop { + if let Ok(stats) = obs.stats(origin_run_id).await { + let body_matches = + assert_origin.active_body.is_none_or(|expected| stats.active_body_connections == expected); + if body_matches || tokio::time::Instant::now() >= deadline { + if let Some(max_tcp) = assert_origin.max_active_tcp { + if stats.max_active_tcp_connections > max_tcp { + *any_failed = true; + events.push((step_id, "origin_peak_tcp_exceeded")); + } + } + if let Some(max_body) = assert_origin.max_active_body { + if stats.max_active_body_connections > max_body { + *any_failed = true; + events.push((step_id, "origin_peak_body_exceeded")); + } + } + if let Some(act_body) = assert_origin.active_body { + if stats.active_body_connections != act_body { + *any_failed = true; + events.push((step_id, "origin_active_body_mismatch")); + } + } + let check_evictions = assert_origin.no_evictions.unwrap_or(false); + let check_rejections = assert_origin.no_limit_rejections.unwrap_or(false); + if check_evictions || check_rejections { + if let Ok(evts) = obs.events(origin_run_id).await { + if check_evictions + && evts.iter().any(|e| matches!(e.kind, OriginEventKind::EvictionTriggered { .. })) + { + *any_failed = true; + events.push((step_id, "origin_eviction_observed")); + } + if check_rejections + && evts.iter().any(|e| matches!(e.kind, OriginEventKind::LimitRejected { .. })) + { + *any_failed = true; + events.push((step_id, "origin_rejection_observed")); + } + } else { + *observations_incomplete = true; + events.push((step_id, "origin_events_unavailable")); + } + } + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } else { + if tokio::time::Instant::now() >= deadline { + *observations_incomplete = true; + events.push((step_id, "origin_stats_unavailable")); + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + } +} + +/// Verifies the SUT's user-side accounting (`active_users`, `active_user_connections`) +/// after a step, polling briefly so an in-flight cleanup does not fail the check. +async fn check_runtime_assertions<'a>( + step_id: &'a str, + assertions: &RuntimeAssertions, + observer: &TuliproxObserver, + events: &mut Vec<(&'a str, &'static str)>, + any_failed: &mut bool, + observations_incomplete: &mut bool, +) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + loop { + if let Ok(status) = observer.status().await { + let users = status.get("active_users").and_then(serde_json::Value::as_u64); + let connections = status.get("active_user_connections").and_then(serde_json::Value::as_u64); + let (Some(users), Some(connections)) = (users, connections) else { + *observations_incomplete = true; + events.push((step_id, "runtime_user_counts_unavailable")); + return; + }; + let users_ok = assertions.active_users.is_none_or(|expected| users == expected as u64); + let connections_ok = + assertions.active_user_connections.is_none_or(|expected| connections == expected as u64); + if (users_ok && connections_ok) || tokio::time::Instant::now() >= deadline { + if !users_ok { + *any_failed = true; + events.push((step_id, "runtime_active_users_mismatch")); + } + if !connections_ok { + *any_failed = true; + events.push((step_id, "runtime_active_user_connections_mismatch")); + } + return; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } else if tokio::time::Instant::now() >= deadline { + *observations_incomplete = true; + events.push((step_id, "runtime_status_unavailable")); + return; + } else { + tokio::time::sleep(Duration::from_millis(25)).await; + } + } +} + +#[allow(clippy::too_many_lines)] +async fn execute_scenario_steps<'a>( + scenario: &'a Scenario, + steps: &'a [tuliprox_testkit::config::Step], + observer: &TuliproxObserver, + origin_observer: Option<&OriginObserver>, + controller_state: &ControllerState, + virtual_ids_by_user: &HashMap, +) -> Result<(bool, bool, Vec<(&'a str, &'static str)>), TestkitError> { + let mut any_failed = false; + let mut observations_incomplete = false; + let mut events = Vec::new(); + let mut held_playbacks: HashMap = HashMap::new(); + let mut remote_playbacks: HashMap = HashMap::new(); + let mut expected_terminations = HashSet::new(); + let mut playback_users = HashMap::new(); + let mut admission_oracles = scenario.policy_contract.as_ref().map(|contract| { + contract + .users + .iter() + .map(|(username, policy)| { + ( + username.clone(), + AdmissionOracle::with_policy( + policy.max_connections, + policy.soft_connections, + contract.effective_admission_strategies(), + contract.grace.as_ref().map(|grace| grace.mode), + ), + ) + }) + .collect::>() + }); + let mut admission_order = 0_u64; + if let Some(obs) = origin_observer { + let origin_run_id = + scenario.origin.as_ref().map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + let _ = obs.reset(&origin_run_id.0).await; + } + let execution_result = async { + for step in steps { + if let Some(stop) = &step.stop { + if let Some(oracles) = admission_oracles.as_mut() { + if let Some(username) = playback_users.remove(&stop.playback_id) { + oracles + .get_mut(&username) + .ok_or_else(|| TestkitError::Protocol(format!("missing admission oracle for {username}")))? + .release(&stop.playback_id)?; + } + } + if let Some(held) = held_playbacks.remove(&stop.playback_id) { + let _ = held.release.send(()); + match held.task.await { + Ok(Ok(PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. })) => { + events.push((stop.playback_id.as_str(), "stopped")); + } + Ok(Ok(_) | Err(_)) | Err(_) => any_failed = true, + } + } else if let Some(agent) = remote_playbacks.remove(&stop.playback_id) { + let outcome = dispatch_remote_stop( + controller_state, + &agent, + &scenario.name, + &step.command_id, + &stop.playback_id, + ) + .await?; + match outcome { + PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. } => { + events.push((stop.playback_id.as_str(), "stopped")); + } + _ => any_failed = true, + } + } else { + return Err(TestkitError::Protocol(format!( + "playback {} was not active at stop", + stop.playback_id + ))); + } + + if let Some(assert_origin) = &step.assert_origin { + if let Some(obs) = origin_observer { + let origin_run_id = scenario + .origin + .as_ref() + .map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + check_origin_assertions( + stop.playback_id.as_str(), + assert_origin, + obs, + &origin_run_id.0, + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + } else { + observations_incomplete = true; + events.push((stop.playback_id.as_str(), "origin_observer_missing")); + } + } + + if let Some(runtime) = &step.assert_runtime { + check_runtime_assertions( + stop.playback_id.as_str(), + runtime, + observer, + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + } + + continue; + } + let start = + step.start.as_ref().ok_or_else(|| TestkitError::Configuration("step has no start".to_owned()))?; + let actor = scenario + .actors + .iter() + .find(|actor| actor.id == start.actor) + .ok_or_else(|| TestkitError::Configuration(format!("unknown actor {}", start.actor)))?; + let is_vod = start.vod_object.is_some() + || start.range.is_some() + || start.read_limit_bytes.is_some() + || start.method.is_some(); + let frames = step + .await_frames + .or_else(|| step.await_condition.as_ref().and_then(|condition| condition.valid_frames)) + .unwrap_or(1); + let url_owned: String; + let url = if let Some(url) = start.url.as_deref() { + url + } else { + let vids = if let Some(username) = actor.username.as_deref() { + virtual_ids_by_user.get(username).or_else(|| virtual_ids_by_user.get("")) + } else { + virtual_ids_by_user.get("") + } + .ok_or_else(|| { + TestkitError::Configuration(format!( + "virtual-ID discovery was not initialized for actor {}", + actor.id + )) + })?; + if let Some(vod) = start.vod_object.as_deref() { + vids.named_playback_url(vod)? + } else { + let channel = start.channel.as_deref().ok_or_else(|| { + TestkitError::Configuration("start has no URL, vod_object, or channel".to_owned()) + })?; + let channel_def = scenario + .channels + .get(channel) + .ok_or_else(|| TestkitError::Configuration(format!("unknown channel {channel}")))?; + let marker = channel_def.origin_marker; + url_owned = resolve_playback_url( + scenario.tuliprox.playback_endpoint, + &channel_def.protocol, + &scenario.tuliprox.base_url, + actor.username.as_deref().unwrap_or(""), + fixture_password(), + vids, + marker, + )?; + &url_owned + } + }; + let marker = if is_vod { + 0 + } else { + start + .channel + .as_deref() + .and_then(|channel| scenario.channels.get(channel)) + .map_or_else(|| marker_from_url(url), |channel| Ok(channel.origin_marker))? + }; + admission_order = admission_order.saturating_add(1); + let mut evicted_victim = None; + let mut evicted_victim_id = None; + if let Some(oracles) = admission_oracles.as_mut() { + let username = actor.username.as_deref().ok_or_else(|| { + TestkitError::Configuration(format!( + "actor {} requires username because this scenario has a user admission policy", + actor.id + )) + })?; + let oracle = oracles.get_mut(username).ok_or_else(|| { + TestkitError::Configuration(format!( + "actor {} references user {username} absent from policy_contract", + actor.id + )) + })?; + let client_ip = + actor.client_ip.as_ref().and_then(|ip| ip.value.as_deref()).unwrap_or("peer-address").to_owned(); + let (decision, evicted) = oracle.admit_with_evicted(AdmissionRequest { + playback_id: start.playback_id.clone(), + client_ip, + started_order: admission_order, + })?; + evicted_victim = evicted; + let oracle_expected = match &decision { + AdmissionDecision::Admit + | AdmissionDecision::AdmitSoft + | AdmissionDecision::Evict { .. } + | AdmissionDecision::Grace { .. } => ExpectedPlayback::Streaming, + AdmissionDecision::Deny => ExpectedPlayback::Rejected, + }; + let oracle_tracks_playback = matches!( + &decision, + AdmissionDecision::Admit | AdmissionDecision::AdmitSoft | AdmissionDecision::Evict { .. } + ); + if let AdmissionDecision::Evict { playback_id } = &decision { + expected_terminations.insert(playback_id.clone()); + evicted_victim_id = Some(playback_id.clone()); + } + let provider_limited_rejection = + scenario.policy_contract.as_ref().and_then(|contract| contract.provider_max_connections).is_some() + && step.expect.is_rejected() + && oracle_expected == ExpectedPlayback::Streaming; + // The static oracle does not model the time-based reentry guard, so a + // suppressed retry intentionally diverges from its eviction prediction. + // The concrete SUT outcome is asserted through `step.expect`. + let reentry_suppression = matches!(step.expect, ExpectedPlayback::Suppressed); + if step.expect != oracle_expected && !provider_limited_rejection && !reentry_suppression { + return Err(TestkitError::Configuration(format!( + "step {} expects {:?}, but policy oracle expects {:?}", + step.command_id, step.expect, oracle_expected + ))); + } + if oracle_tracks_playback { + playback_users.insert(start.playback_id.clone(), username.to_owned()); + } + } + let mut headers = actor_request_headers(actor, actor.agent == "local")?; + if let Some(ua) = &start.user_agent { + headers.insert("user-agent".to_owned(), ua.clone()); + } + let origin_run_id = scenario + .origin + .as_ref() + .map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + let local_outcome = if actor.agent == "local" { + if is_vod { + match start_held_vod_playback( + start.playback_id.clone(), + url, + start.vod_object.clone(), + start.method.clone(), + start.range.clone(), + start.expected_status, + start.expected_content_range.clone(), + start.read_limit_bytes, + start.post_read_action.unwrap_or_default(), + &headers, + ) + .await? + { + Ok(held) => { + held_playbacks.insert(start.playback_id.clone(), held); + Some(PlaybackOutcome::Streaming { frames: 0, bytes: 0 }) + } + Err(outcome) => Some(outcome), + } + } else { + match start_held_playback(start.playback_id.clone(), &origin_run_id, url, marker, frames, &headers) + .await? + { + Ok(held) => { + held_playbacks.insert(start.playback_id.clone(), held); + Some(PlaybackOutcome::Streaming { frames, bytes: 0 }) + } + Err(outcome) => Some(outcome), + } + } + } else { + None + }; + let remote_outcome = if actor.agent == "local" { + None + } else { + dispatch_remote_playback( + controller_state, + &actor.agent, + &scenario.name, + &step.command_id, + &start.playback_id, + url, + headers.clone(), + Some(origin_run_id), + Some(marker), + Some(frames), + ) + .await? + }; + let (step_passed, actual_streaming) = match (&local_outcome, &remote_outcome) { + (Some(outcome), _) | (None, Some(outcome)) => { + (step.expect.matches_outcome(outcome), outcome.is_streaming()) + } + (None, None) => (false, false), + }; + if step_passed { + events.push(( + start.playback_id.as_str(), + if step.expect.is_streaming() { "received_valid_frames" } else { "rejected_as_expected" }, + )); + if remote_outcome.as_ref().is_some_and(PlaybackOutcome::is_streaming) { + remote_playbacks.insert(start.playback_id.clone(), actor.agent.clone()); + } + } else { + eprintln!( + "Step {} ({}) failed: expect={:?}, local_outcome={:?}, remote_outcome={:?}", + step.command_id, start.playback_id, step.expect, local_outcome, remote_outcome + ); + any_failed = true; + events.push((start.playback_id.as_str(), "failed")); + } + if !actual_streaming { + if let Some(oracles) = admission_oracles.as_mut() { + if let Some(username) = actor.username.as_deref() { + if let Some(oracle) = oracles.get_mut(username) { + oracle.rollback(&start.playback_id, evicted_victim); + } + } + playback_users.remove(&start.playback_id); + if let Some(victim_id) = &evicted_victim_id { + expected_terminations.remove(victim_id); + } + } + } + if let Some(assert_origin) = &step.assert_origin { + if let Some(obs) = origin_observer { + let origin_run_id = scenario + .origin + .as_ref() + .map_or_else(|| RunId::new(&scenario.name), |origin| RunId::new(&origin.run_id)); + check_origin_assertions( + start.playback_id.as_str(), + assert_origin, + obs, + &origin_run_id.0, + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + } else { + observations_incomplete = true; + events.push((start.playback_id.as_str(), "origin_observer_missing")); + } + } + if let Ok(snapshot) = observer.runtime_snapshot().await { + events.push((start.playback_id.as_str(), "runtime_observed")); + if let Some(policy) = &scenario.policy_contract { + if let Ok(slots) = provider_slot_count(&snapshot.status) { + if policy.provider_max_connections.is_some_and(|limit| slots > usize::from(limit)) { + any_failed = true; + events.push((start.playback_id.as_str(), "provider_limit_exceeded")); + } + if policy.expected_provider_slots.is_some_and(|expected| slots != expected) { + any_failed = true; + events.push((start.playback_id.as_str(), "provider_slot_count_mismatch")); + } + } else { + observations_incomplete = true; + events.push((start.playback_id.as_str(), "provider_observation_invalid")); + } + } + } else { + observations_incomplete = true; + events.push((start.playback_id.as_str(), "runtime_observation_unavailable")); + } + if let Some(runtime) = &step.assert_runtime { + check_runtime_assertions( + start.playback_id.as_str(), + runtime, + observer, + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + } + } + Ok::<(), TestkitError>(()) + } + .await; + for (_, mut held) in held_playbacks { + let expected_termination = expected_terminations.contains(&held.playback_id); + let event_playback_id = steps + .iter() + .filter_map(|step| step.start.as_ref()) + .find(|start| start.playback_id == held.playback_id) + .map(|start| start.playback_id.as_str()) + .ok_or_else(|| TestkitError::Protocol(format!("missing scenario playback {}", held.playback_id)))?; + if expected_termination { + match tokio::time::timeout(PLAYBACK_IDLE_TIMEOUT, &mut held.task).await { + Ok(Ok(Ok(PlaybackOutcome::Streaming { .. } | PlaybackOutcome::ExplicitStop))) => { + events.push((event_playback_id, "unexpected_clean_terminal")); + any_failed = true; + } + Ok(Ok(Ok(_))) => events.push((event_playback_id, "evicted")), + Ok(Ok(Err(_)) | Err(_)) => { + events.push((event_playback_id, "unexpected_eviction_terminal")); + any_failed = true; + } + Err(_) => { + held.task.abort(); + let _ = held.task.await; + any_failed = true; + events.push((event_playback_id, "expected_eviction_not_observed")); + } + } + let _ = held.release.is_closed(); + } else { + let _ = held.release.send(()); + match held.task.await { + Ok(Ok(PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. })) => {} + Ok(Ok(_) | Err(_)) | Err(_) => any_failed = true, + } + } + } + for (playback_id, agent) in remote_playbacks { + if let Ok(outcome) = dispatch_remote_stop( + controller_state, + &agent, + &scenario.name, + &format!("cleanup-{playback_id}"), + &playback_id, + ) + .await + { + match outcome { + PlaybackOutcome::ExplicitStop | PlaybackOutcome::Streaming { .. } => {} + _ => any_failed = true, + } + } else { + any_failed = true; + } + } + execution_result?; + if scenario.policy_contract.as_ref().is_some_and(|contract| contract.provider_max_connections.is_some()) { + if wait_for_provider_slots(observer, 0, Duration::from_secs(5)).await.is_ok() { + events.push((scenario.name.as_str(), "provider_slots_released")); + } else { + any_failed = true; + events.push((scenario.name.as_str(), "provider_slots_not_released")); + } + } + Ok((any_failed, observations_incomplete, events)) +} + +async fn wait_for_provider_slots( + observer: &TuliproxObserver, + expected: usize, + timeout: Duration, +) -> Result<(), TestkitError> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if provider_slot_count(&observer.status().await?)? == expected { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!("provider slots did not converge to {expected}"))); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +fn provider_slot_count(status: &serde_json::Value) -> Result { + let providers = status.get("active_provider_connections").ok_or_else(|| { + TestkitError::Protocol("status response missing active_provider_connections field".to_owned()) + })?; + if providers.is_null() { + return Ok(0); + } + let Some(providers) = providers.as_object() else { + return Err(TestkitError::Protocol("status active_provider_connections is not an object".to_owned())); + }; + providers.values().try_fold(0_usize, |total, value| { + let count = value + .as_u64() + .ok_or_else(|| TestkitError::Protocol("provider connection count is not an unsigned integer".to_owned()))?; + let count = usize::try_from(count) + .map_err(|_| TestkitError::Protocol("provider connection count exceeds platform capacity".to_owned()))?; + total.checked_add(count).ok_or_else(|| TestkitError::Protocol("provider connection count overflow".to_owned())) + }) +} + +fn actor_request_headers( + actor: &tuliprox_testkit::config::Actor, + allow_controller_secrets: bool, +) -> Result, TestkitError> { + let mut headers = BTreeMap::new(); + if let Some(user_agent) = &actor.user_agent { + headers.insert("user-agent".to_owned(), user_agent.clone()); + } + if let Some(client_ip) = &actor.client_ip { + let value = client_ip.value.as_deref().filter(|value| !value.trim().is_empty()).ok_or_else(|| { + TestkitError::Configuration(format!("actor {} client_ip requires a non-empty value", actor.id)) + })?; + match client_ip.mode.as_str() { + "x-real-ip" => { + headers.insert("x-real-ip".to_owned(), value.to_owned()); + } + "forwarded" | "x-forwarded-for" => { + headers.insert("x-forwarded-for".to_owned(), value.to_owned()); + } + unsupported => { + return Err(TestkitError::Configuration(format!( + "actor {} has unsupported client_ip mode {unsupported}", + actor.id + ))); + } + } + } + match (&actor.username, &actor.password_env) { + (Some(username), Some(password_env)) if allow_controller_secrets => { + let credentials = format!("{username}:{}", environment(password_env)?); + headers.insert("authorization".to_owned(), format!("Basic {}", BASE64.encode(credentials))); + } + (Some(_), Some(_)) => { + return Err(TestkitError::Configuration(format!( + "remote actor {} credentials must be supplied by an agent-local secret provider", + actor.id + ))); + } + (Some(_) | None, None) => {} + _ => { + return Err(TestkitError::Configuration(format!( + "actor {} must specify username and password_env together", + actor.id + ))); + } + } + Ok(headers) +} + +async fn start_controller_transport(agent_listen: SocketAddr, run_name: &str) -> Result { + let listener = TcpListener::bind(agent_listen).await?; + let state: ControllerState = Arc::new(SharedControllerState::new(RunId::new(run_name), 1)); + let app = Router::new().route("/agent", get(controller_websocket)).with_state(state.clone()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok(state) +} + +async fn wait_for_required_agents(controller_state: &ControllerState, required: &[String]) -> Result<(), TestkitError> { + if required.is_empty() { + return Ok(()); + } + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let connected = controller_state + .ready_agents() + .await + .into_iter() + .map(|agent| agent.0) + .collect::>(); + if required.iter().all(|agent| connected.contains(agent)) { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol("required test agents did not become ready before deadline".to_owned())); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[allow(clippy::too_many_arguments)] +async fn dispatch_remote_playback( + controller_state: &ControllerState, + agent: &str, + run_name: &str, + command_id: &str, + playback_id: &str, + url: &str, + headers: BTreeMap, + expected_run_id: Option, + expected_marker: Option, + required_frames: Option, +) -> Result, TestkitError> { + let command = Envelope { + schema_version: 1, + run_id: RunId::new(run_name), + run_generation: 1, + message_id: command_id.to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: controller_state.next_command_sequence(), + caused_by_command_id: Some(tuliprox_testkit::protocol::CommandId::new(command_id)), + local_elapsed_nanos: 0, + payload: ControlCommand::StartPlayback { + playback_id: tuliprox_testkit::protocol::PlaybackId::new(playback_id), + url: url.to_owned(), + headers, + expected_run_id, + expected_marker, + required_frames, + }, + }; + controller_state.dispatch(&AgentId::new(agent), command).await?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some(event) = controller_state.take_playback_event(playback_id).await { + match event { + PlaybackEvent::FirstValidFrame { .. } => { + return Ok(Some(PlaybackOutcome::Streaming { frames: required_frames.unwrap_or(5), bytes: 0 })); + } + PlaybackEvent::Terminal { typed_outcome, outcome, .. } => { + return Ok(Some(typed_outcome.unwrap_or_else(|| match outcome.as_str() { + "passed" => PlaybackOutcome::Streaming { frames: required_frames.unwrap_or(5), bytes: 0 }, + "rejected" | "rejected_as_expected" => { + PlaybackOutcome::AdmissionRejected { reason: RejectionReason::Other("rejected".to_owned()) } + } + _ => PlaybackOutcome::InfrastructureError { message: outcome }, + }))); + } + PlaybackEvent::CommandAccepted { .. } + | PlaybackEvent::HeadersReceived { .. } + | PlaybackEvent::Progress { .. } + | PlaybackEvent::LeaseExpired => {} + } + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!( + "remote agent {agent} did not report terminal playback {playback_id}" + ))); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn dispatch_remote_stop( + controller_state: &ControllerState, + agent: &str, + run_name: &str, + command_id: &str, + playback_id: &str, +) -> Result { + controller_state + .dispatch( + &AgentId::new(agent), + Envelope { + schema_version: 1, + run_id: RunId::new(run_name), + run_generation: 1, + message_id: command_id.to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: controller_state.next_command_sequence(), + caused_by_command_id: Some(tuliprox_testkit::protocol::CommandId::new(command_id)), + local_elapsed_nanos: 0, + payload: ControlCommand::StopPlayback { + playback_id: tuliprox_testkit::protocol::PlaybackId::new(playback_id), + }, + }, + ) + .await?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some(PlaybackEvent::Terminal { typed_outcome, outcome, .. }) = + controller_state.take_playback_event(playback_id).await + { + return Ok(typed_outcome.unwrap_or(match outcome.as_str() { + "passed" | "cancelled" => PlaybackOutcome::ExplicitStop, + _ => PlaybackOutcome::InfrastructureError { message: outcome }, + })); + } + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol(format!( + "remote agent {agent} did not confirm stop for playback {playback_id}" + ))); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[allow(clippy::too_many_arguments)] +fn write_raw_report( + scenario_name: &str, + directory: &std::path::Path, + outcome: RunExit, + events: Vec<(&str, &str)>, + policy_hash: &str, + run_id: Option<&str>, + error_kind: Option<&str>, + error_message: Option<&str>, + exit_code: Option, +) -> Result<(), TestkitError> { + let outcome_text = match outcome { + RunExit::Passed => "passed", + RunExit::Failed => "failed", + RunExit::Invalid => "invalid", + RunExit::Inconclusive => "inconclusive", + }; + let report = RunReport { + schema_version: 1, + scenario: scenario_name, + outcome: outcome_text, + policy_hash, + run_id, + error_kind, + error_message, + exit_code, + events: events + .into_iter() + .map(|(playback_id, event)| ReportEvent { source: "agent", playback_id, event, detail: "redacted" }) + .collect(), + }; + report.write_to(directory) +} + +#[allow(clippy::too_many_arguments)] +fn write_report( + scenario: &Scenario, + directory: &std::path::Path, + outcome: RunExit, + events: Vec<(&str, &str)>, + policy_hash: &str, + run_id: Option<&str>, + error_kind: Option<&str>, + error_message: Option<&str>, + exit_code: Option, +) -> Result<(), TestkitError> { + write_raw_report( + &scenario.name, + directory, + outcome, + events, + policy_hash, + run_id, + error_kind, + error_message, + exit_code, + ) +} + +fn marker_from_url(url: &str) -> Result { + let parsed = + url::Url::parse(url).map_err(|error| TestkitError::Configuration(format!("invalid stream URL: {error}")))?; + let path = parsed + .path_segments() + .ok_or_else(|| TestkitError::Configuration("stream URL has no path segments".to_owned()))? + .collect::>(); + path.iter() + .rev() + .find_map(|segment| segment.split('.').next().and_then(|value| value.parse::().ok())) + .ok_or_else(|| TestkitError::Configuration("stream URL has no numeric marker path segment".to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_origin_state(run_id: &str) -> OriginState { + OriginState { + run_id: RunId::new(run_id), + bitrate: 64_000, + markers: Arc::new(vec![17]), + stream_counter: Arc::new(Mutex::new(4)), + observations: Arc::new(Mutex::new(Vec::new())), + faults: Arc::new(Mutex::new(FaultSchedule::default())), + hls_sequences: Arc::new(Mutex::new(HashMap::from([(17, 9)]))), + tracker: OriginTracker::new(run_id, OriginPolicy::default()), + } + } + + #[test] + fn extracts_marker_from_ts_url() { + assert!(matches!(marker_from_url("http://origin/live/17.ts"), Ok(17))); + } + + #[test] + fn extracts_marker_from_hls_manifest_url() { + assert!(matches!(marker_from_url("http://origin/hls/17/index.m3u8"), Ok(17))); + } + + #[test] + fn recognizes_hls_manifest_urls() { + assert!(is_hls_manifest_url("http://origin/hls/17/index.m3u8")); + assert!(!is_hls_manifest_url("http://origin/live/17.ts")); + } + + #[test] + fn parses_open_and_closed_byte_ranges() { + assert_eq!(tuliprox_testkit::vod::parse_range_header("bytes=2-4", 10), Some((2, 5))); + assert_eq!(tuliprox_testkit::vod::parse_range_header("bytes=8-", 10), Some((8, 10))); + assert_eq!(tuliprox_testkit::vod::parse_range_header("bytes=10-", 10), None); + } + + #[test] + fn vod_validators_are_stable_for_identical_payloads() { + let payload = vod_payload("movie.ts"); + let mut first = StatusCode::OK.into_response(); + let mut second = StatusCode::OK.into_response(); + apply_vod_validators(&mut first, &payload); + apply_vod_validators(&mut second, &payload); + assert_eq!(first.headers().get(header::ETAG), second.headers().get(header::ETAG)); + assert_eq!( + first.headers().get(header::LAST_MODIFIED), + Some(&HeaderValue::from_static("Thu, 01 Jan 1970 00:00:00 GMT")) + ); + } + + #[tokio::test] + async fn origin_reset_is_scoped_to_its_run() { + let state = test_origin_state("run-a"); + state.observations.lock().await.push(OriginObservation { + origin_stream_id: "origin-4".to_owned(), + channel_marker: 17, + frames_emitted: 0, + bytes_emitted: 0, + }); + assert_eq!(reset_origin_run(Path("other".to_owned()), State(state.clone())).await, StatusCode::NOT_FOUND); + assert_eq!(reset_origin_run(Path("run-a".to_owned()), State(state.clone())).await, StatusCode::NO_CONTENT); + assert!(state.observations.lock().await.is_empty()); + assert_eq!(*state.stream_counter.lock().await, 0); + assert!(state.hls_sequences.lock().await.is_empty()); + } + + #[tokio::test] + async fn catalog_exposes_each_configured_marker() { + let mut state = test_origin_state("catalog-run"); + state.markers = Arc::new(vec![17, 19]); + let document = catalog_m3u(&state, "127.0.0.1"); + assert!(document.contains("tvg-id=\"test-17\"")); + assert!(document.contains("tvg-id=\"test-19\"")); + assert!(document.contains("run=catalog-run")); + } + + #[tokio::test] + async fn hls_rejects_a_marker_absent_from_the_origin_catalog() { + let state = test_origin_state("hls-run"); + assert_eq!( + hls( + Path("19/index.m3u8".to_owned()), + Query(HashMap::from([("run".to_owned(), "hls-run".to_owned())])), + State(state), + ) + .await + .status(), + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn live_rejects_a_missing_or_foreign_run_correlation() { + let state = test_origin_state("live-run"); + assert_eq!( + live(Path("17.ts".to_owned()), Query(HashMap::new()), State(state.clone()), None, HeaderMap::new()) + .await + .status(), + StatusCode::NOT_FOUND + ); + assert_eq!( + live( + Path("17.ts".to_owned()), + Query(HashMap::from([("run".to_owned(), "other".to_owned())])), + State(state), + None, + HeaderMap::new(), + ) + .await + .status(), + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn vod_head_rejects_a_missing_run_correlation() { + let state = test_origin_state("vod-run"); + assert_eq!( + vod_head(Path("movie.ts".to_owned()), Query(HashMap::new()), State(state)).await.status(), + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn faults_cannot_be_mutated_by_a_foreign_run() { + let state = test_origin_state("fault-run"); + assert_eq!( + set_fault(Path(("other".to_owned(), "close-before-first-byte".to_owned())), State(state.clone()),).await, + StatusCode::NOT_FOUND + ); + assert!(state.faults.lock().await.take("close-before-first-byte").is_none()); + } + + #[test] + fn actor_identity_uses_configured_forwarded_ip_and_user_agent() { + let actor = tuliprox_testkit::config::Actor { + id: "viewer".to_owned(), + agent: "local".to_owned(), + username: None, + password_env: None, + user_agent: Some("test-receiver/1".to_owned()), + client_ip: Some(tuliprox_testkit::config::ClientIp { + mode: "x-real-ip".to_owned(), + value: Some("198.51.100.42".to_owned()), + }), + }; + let headers = actor_request_headers(&actor, true).unwrap(); + assert_eq!(headers.get("x-real-ip"), Some(&"198.51.100.42".to_owned())); + assert_eq!(headers.get("user-agent"), Some(&"test-receiver/1".to_owned())); + } + + #[test] + fn remote_actor_cannot_receive_controller_credentials() { + let actor = tuliprox_testkit::config::Actor { + id: "viewer".to_owned(), + agent: "agent-b".to_owned(), + username: Some("alice".to_owned()), + password_env: Some("TESTKIT_PASSWORD".to_owned()), + user_agent: None, + client_ip: None, + }; + assert!(matches!(actor_request_headers(&actor, false), Err(TestkitError::Configuration(_)))); + } + + #[tokio::test] + async fn vod_reader_rejects_206_missing_content_range() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buf = [0u8; 1024]; + let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut buf).await; + let response = "HTTP/1.1 206 Partial Content\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntest"; + let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await; + }); + + let (ready_tx, _ready_rx) = tokio::sync::oneshot::channel(); + let (_release_tx, release_rx) = tokio::sync::oneshot::channel(); + let outcome = run_vod_until_released( + format!("http://127.0.0.1:{port}/movie.mkv"), + None, + None, + Some("bytes=0-3".to_owned()), + None, + None, + None, + PostReadAction::KeepOpen, + BTreeMap::new(), + ready_tx, + release_rx, + ) + .await + .unwrap(); + + assert!(matches!(outcome, PlaybackOutcome::InvalidData { message } if message.contains("omits Content-Range"))); + } + + #[tokio::test] + async fn check_origin_assertions_events_http_500_marks_observations_incomplete() { + use axum::routing::get; + let app = Router::new() + .route( + "/v1/runs/test-run/stats", + get(|| async { + Json(OriginStats { + active_tcp_connections: 0, + max_active_tcp_connections: 0, + active_body_connections: 0, + max_active_body_connections: 0, + total_tcp_connections: 0, + total_requests: 0, + total_bytes_emitted: 0, + }) + }), + ) + .route("/v1/runs/test-run/events", get(|| async { StatusCode::INTERNAL_SERVER_ERROR })); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let obs = OriginObserver::new(&format!("http://127.0.0.1:{port}")); + let mut events = Vec::new(); + let mut any_failed = false; + let mut observations_incomplete = false; + let assert_origin = tuliprox_testkit::config::AssertOrigin { + active_body: None, + max_active_body: None, + max_active_tcp: None, + no_evictions: Some(true), + no_limit_rejections: None, + }; + + check_origin_assertions( + "step-1", + &assert_origin, + &obs, + "test-run", + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + + assert!(!any_failed); + assert!(observations_incomplete); + assert!(events.iter().any(|(s, e)| *s == "step-1" && *e == "origin_events_unavailable")); + } + + #[tokio::test] + async fn check_origin_assertions_eviction_observed_marks_failed() { + use axum::routing::get; + let app = Router::new() + .route( + "/v1/runs/test-run/stats", + get(|| async { + Json(OriginStats { + active_tcp_connections: 0, + max_active_tcp_connections: 0, + active_body_connections: 0, + max_active_body_connections: 0, + total_tcp_connections: 0, + total_requests: 0, + total_bytes_emitted: 0, + }) + }), + ) + .route( + "/v1/runs/test-run/events", + get(|| async { + Json(vec![OriginEvent::new( + 1, + "test-run", + OriginEventKind::EvictionTriggered { + evicted_conn_id: 1, + evicted_request_id: 1, + triggering_conn_id: 2, + }, + )]) + }), + ); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let obs = OriginObserver::new(&format!("http://127.0.0.1:{port}")); + let mut events = Vec::new(); + let mut any_failed = false; + let mut observations_incomplete = false; + let assert_origin = tuliprox_testkit::config::AssertOrigin { + active_body: None, + max_active_body: None, + max_active_tcp: None, + no_evictions: Some(true), + no_limit_rejections: None, + }; + + check_origin_assertions( + "step-1", + &assert_origin, + &obs, + "test-run", + &mut events, + &mut any_failed, + &mut observations_incomplete, + ) + .await; + + assert!(any_failed); + assert!(!observations_incomplete); + assert!(events.iter().any(|(s, e)| *s == "step-1" && *e == "origin_eviction_observed")); + } +} diff --git a/tools/tuliprox-testkit/src/observation.rs b/tools/tuliprox-testkit/src/observation.rs new file mode 100644 index 000000000..bba8f843b --- /dev/null +++ b/tools/tuliprox-testkit/src/observation.rs @@ -0,0 +1,49 @@ +use crate::TestkitError; +use serde_json::Value; + +#[derive(Debug, Clone)] +pub struct RuntimeSnapshot { + pub status: Value, + pub streams: Value, + pub history: Value, +} + +#[derive(Clone)] +pub struct TuliproxObserver { + base_url: String, + client: reqwest::Client, + credentials: Option<(String, String)>, +} + +impl TuliproxObserver { + #[must_use] + pub fn new(base_url: impl Into, credentials: Option<(String, String)>) -> Self { + Self { base_url: base_url.into().trim_end_matches('/').to_owned(), client: reqwest::Client::new(), credentials } + } + + pub async fn config(&self) -> Result { self.get_json("config").await } + + pub async fn status(&self) -> Result { self.get_json("status").await } + + pub async fn streams(&self) -> Result { self.get_json("streams").await } + + pub async fn stream_history(&self) -> Result { self.get_json("stream-history").await } + + pub async fn runtime_snapshot(&self) -> Result { + let (status, streams) = tokio::join!(self.status(), self.streams()); + Ok(RuntimeSnapshot { status: status?, streams: streams?, history: Value::Null }) + } + + async fn get_json(&self, endpoint: &str) -> Result { + let request = self.client.get(format!("{}/{}", self.base_url, endpoint)); + let request = match &self.credentials { + Some((username, password)) => request.basic_auth(username, Some(password)), + None => request, + }; + let response = request.send().await?; + if !response.status().is_success() { + return Err(TestkitError::Protocol(format!("observer endpoint {endpoint} returned {}", response.status()))); + } + response.json().await.map_err(TestkitError::Http) + } +} diff --git a/tools/tuliprox-testkit/src/oracle.rs b/tools/tuliprox-testkit/src/oracle.rs new file mode 100644 index 000000000..71f13b06e --- /dev/null +++ b/tools/tuliprox-testkit/src/oracle.rs @@ -0,0 +1,293 @@ +use crate::TestkitError; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AdmissionStrategy { + EvictUserSameIpOldest, + EvictUserSameIpLatest, + EvictUserOldest, + EvictUserLatest, + GraceInstantStream, + GraceHoldStream, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GraceMode { + Instant, + HoldStream, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Request { + pub playback_id: String, + pub client_ip: String, + pub started_order: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decision { + Admit, + AdmitSoft, + Deny, + Evict { playback_id: String }, + Grace { mode: GraceMode }, +} + +#[derive(Debug)] +pub struct AdmissionOracle { + max_connections: usize, + soft_connections: usize, + strategies: Vec, + grace: Option, + active: HashMap, +} + +impl AdmissionOracle { + #[must_use] + pub fn new(max_connections: usize, strategies: Vec) -> Self { + Self { max_connections, soft_connections: 0, strategies, grace: None, active: HashMap::new() } + } + + #[must_use] + pub fn with_policy( + max_connections: usize, + soft_connections: usize, + strategies: Vec, + grace: Option, + ) -> Self { + Self { max_connections, soft_connections, strategies, grace, active: HashMap::new() } + } + + pub fn admit_with_evicted(&mut self, request: Request) -> Result<(Decision, Option), TestkitError> { + if self.active.contains_key(&request.playback_id) { + return Err(TestkitError::Configuration("duplicate oracle playback".to_owned())); + } + if self.active.len() < self.max_connections { + self.active.insert(request.playback_id.clone(), request); + return Ok((Decision::Admit, None)); + } + if self.active.len() < self.max_connections.saturating_add(self.soft_connections) { + self.active.insert(request.playback_id.clone(), request); + return Ok((Decision::AdmitSoft, None)); + } + for strategy in &self.strategies { + if let Some(mode) = grace_mode(*strategy) { + return Ok((Decision::Grace { mode }, None)); + } + let candidate = self.select(&request, *strategy)?; + if let Some(candidate) = candidate { + let victim = candidate.playback_id.clone(); + let evicted_req = self.active.remove(&victim); + self.active.insert(request.playback_id.clone(), request); + return Ok((Decision::Evict { playback_id: victim }, evicted_req)); + } + } + Ok(( + match self.grace { + Some(mode) => Decision::Grace { mode }, + None => Decision::Deny, + }, + None, + )) + } + + pub fn admit(&mut self, request: Request) -> Result { + self.admit_with_evicted(request).map(|(decision, _)| decision) + } + + pub fn rollback(&mut self, playback_id: &str, evicted: Option) { + self.active.remove(playback_id); + if let Some(restored) = evicted { + self.active.insert(restored.playback_id.clone(), restored); + } + } + + pub fn release(&mut self, playback_id: &str) -> Result<(), TestkitError> { + self.active + .remove(playback_id) + .map(|_| ()) + .ok_or_else(|| TestkitError::Configuration(format!("cannot release inactive playback {playback_id}"))) + } + + fn select(&self, incoming: &Request, strategy: AdmissionStrategy) -> Result, TestkitError> { + if grace_mode(strategy).is_some() { + return Ok(None); + } + let same_ip = + matches!(strategy, AdmissionStrategy::EvictUserSameIpOldest | AdmissionStrategy::EvictUserSameIpLatest); + let latest = matches!(strategy, AdmissionStrategy::EvictUserSameIpLatest | AdmissionStrategy::EvictUserLatest); + let candidates = self.active.values().filter(|candidate| !same_ip || candidate.client_ip == incoming.client_ip); + let selected = if latest { + candidates.max_by_key(|candidate| candidate.started_order) + } else { + candidates.min_by_key(|candidate| candidate.started_order) + }; + let Some(selected) = selected else { return Ok(None) }; + let tied = self.active.values().any(|candidate| { + candidate.playback_id != selected.playback_id + && (!same_ip || candidate.client_ip == incoming.client_ip) + && candidate.started_order == selected.started_order + }); + if tied { + return Err(TestkitError::Protocol("candidate ages are ambiguous at SUT timestamp resolution".to_owned())); + } + Ok(Some(selected)) + } +} + +const fn grace_mode(strategy: AdmissionStrategy) -> Option { + match strategy { + AdmissionStrategy::GraceInstantStream => Some(GraceMode::Instant), + AdmissionStrategy::GraceHoldStream => Some(GraceMode::HoldStream), + AdmissionStrategy::EvictUserSameIpOldest + | AdmissionStrategy::EvictUserSameIpLatest + | AdmissionStrategy::EvictUserOldest + | AdmissionStrategy::EvictUserLatest => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn latest_replaces_latest_candidate() { + let mut oracle = AdmissionOracle::new(2, vec![AdmissionStrategy::EvictUserSameIpLatest]); + for (id, order) in [("a", 1), ("b", 2)] { + assert!(matches!( + oracle.admit(Request { + playback_id: id.to_owned(), + client_ip: "10.0.0.1".to_owned(), + started_order: order + }), + Ok(Decision::Admit) + )); + } + assert!( + matches!(oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 3 }), Ok(Decision::Evict { playback_id }) if playback_id == "b") + ); + } + + #[test] + fn soft_slot_precedes_eviction() { + let mut oracle = + AdmissionOracle::with_policy(1, 1, vec![AdmissionStrategy::EvictUserLatest], Some(GraceMode::HoldStream)); + assert!(matches!( + oracle.admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }), + Ok(Decision::Admit) + )); + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }), + Ok(Decision::AdmitSoft) + )); + assert!( + matches!(oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.3".to_owned(), started_order: 3 }), Ok(Decision::Evict { playback_id }) if playback_id == "b") + ); + oracle.release("a").unwrap(); + oracle.release("c").unwrap(); + assert!(matches!( + oracle.admit(Request { playback_id: "d".to_owned(), client_ip: "10.0.0.4".to_owned(), started_order: 4 }), + Ok(Decision::Admit) + )); + assert!(matches!( + oracle.admit(Request { playback_id: "e".to_owned(), client_ip: "10.0.0.5".to_owned(), started_order: 5 }), + Ok(Decision::AdmitSoft) + )); + } + + #[test] + fn soft_slot_exhausts_before_deny() { + let mut oracle = AdmissionOracle::with_policy(1, 1, Vec::new(), None); + assert!(matches!( + oracle.admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }), + Ok(Decision::Admit) + )); + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 2 }), + Ok(Decision::AdmitSoft) + )); + assert!(matches!( + oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 3 }), + Ok(Decision::Deny) + )); + // Releasing the soft slot frees the extra capacity again. + oracle.release("b").unwrap(); + assert!(matches!( + oracle.admit(Request { playback_id: "d".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 4 }), + Ok(Decision::AdmitSoft) + )); + } + + #[test] + fn grace_applies_when_capacity_has_no_eviction_strategy() { + let mut oracle = AdmissionOracle::with_policy(1, 0, Vec::new(), Some(GraceMode::HoldStream)); + assert!(oracle + .admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }) + .is_ok()); + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }), + Ok(Decision::Grace { mode: GraceMode::HoldStream }) + )); + } + + #[test] + fn grace_strategy_is_evaluated_in_its_configured_order() { + let mut oracle = AdmissionOracle::new(1, vec![AdmissionStrategy::GraceInstantStream]); + assert!(oracle + .admit(Request { playback_id: "a".to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }) + .is_ok()); + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }), + Ok(Decision::Grace { mode: GraceMode::Instant }) + )); + } + + #[test] + fn equal_candidate_ages_are_ambiguous() { + let mut oracle = AdmissionOracle::new(2, vec![AdmissionStrategy::EvictUserOldest]); + for id in ["a", "b"] { + assert!(oracle + .admit(Request { playback_id: id.to_owned(), client_ip: "10.0.0.1".to_owned(), started_order: 1 }) + .is_ok()); + } + assert!(oracle + .admit(Request { playback_id: "c".to_owned(), client_ip: "10.0.0.2".to_owned(), started_order: 2 }) + .is_err()); + } + + #[test] + fn sequential_same_ip_eviction_a_then_b_then_c() { + let mut oracle = AdmissionOracle::with_policy( + 1, + 0, + vec![ + AdmissionStrategy::EvictUserSameIpLatest, + AdmissionStrategy::EvictUserSameIpOldest, + AdmissionStrategy::EvictUserLatest, + AdmissionStrategy::EvictUserOldest, + ], + Some(GraceMode::HoldStream), + ); + // A admitted on first slot + assert!(matches!( + oracle.admit(Request { playback_id: "a".to_owned(), client_ip: "10.20.0.10".to_owned(), started_order: 1 }), + Ok(Decision::Admit) + )); + // B evicts A (same IP, latest = A is latest, so A is evicted) + assert!(matches!( + oracle.admit(Request { playback_id: "b".to_owned(), client_ip: "10.20.0.10".to_owned(), started_order: 2 }), + Ok(Decision::Evict { ref playback_id }) if playback_id == "a" + )); + // C evicts B (same IP, latest = B is latest) + assert!(matches!( + oracle.admit(Request { playback_id: "c".to_owned(), client_ip: "10.20.0.10".to_owned(), started_order: 3 }), + Ok(Decision::Evict { ref playback_id }) if playback_id == "b" + )); + // Only C remains + assert_eq!(oracle.active.len(), 1); + assert!(oracle.active.contains_key("c")); + } +} diff --git a/tools/tuliprox-testkit/src/origin_events.rs b/tools/tuliprox-testkit/src/origin_events.rs new file mode 100644 index 000000000..e60aa9db9 --- /dev/null +++ b/tools/tuliprox-testkit/src/origin_events.rs @@ -0,0 +1,127 @@ +use serde::{Deserialize, Serialize}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BodyCloseReason { + Completed, + ClientDisconnected, + Evicted, + Stalled, + Error(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum OriginEventKind { + TcpAccepted { + conn_id: u64, + remote_addr: String, + }, + RequestStarted { + conn_id: u64, + request_id: u64, + method: String, + path: String, + range: Option, + user_agent: Option, + account: Option, + }, + BodyStarted { + conn_id: u64, + request_id: u64, + content_length: Option, + }, + BodyClosed { + conn_id: u64, + request_id: u64, + bytes_emitted: u64, + duration_ms: u64, + reason: BodyCloseReason, + }, + TcpClosed { + conn_id: u64, + duration_ms: u64, + bytes_read: u64, + bytes_written: u64, + reason: String, + }, + LimitRejected { + conn_id: u64, + request_id: u64, + limit: usize, + current_active: usize, + }, + EvictionTriggered { + evicted_conn_id: u64, + evicted_request_id: u64, + triggering_conn_id: u64, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OriginEvent { + pub seq: u64, + pub timestamp_ns: u128, + pub run_id: String, + #[serde(flatten)] + pub kind: OriginEventKind, +} + +impl OriginEvent { + #[must_use] + pub fn new(seq: u64, run_id: impl Into, kind: OriginEventKind) -> Self { + let timestamp_ns = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |duration| duration.as_nanos()); + Self { seq, timestamp_ns, run_id: run_id.into(), kind } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct OriginStats { + pub active_tcp_connections: usize, + pub max_active_tcp_connections: usize, + pub active_body_connections: usize, + pub max_active_body_connections: usize, + pub total_tcp_connections: u64, + pub total_requests: u64, + pub total_bytes_emitted: u64, +} + +#[must_use] +pub fn redact_path_and_query(raw: &str) -> String { + let Some((path, query)) = raw.split_once('?') else { + return raw.to_owned(); + }; + let redacted_query: Vec = query + .split('&') + .map(|pair| { + if let Some((key, _)) = pair.split_once('=') { + let lower = key.to_ascii_lowercase(); + if lower.contains("pass") + || lower.contains("token") + || lower.contains("secret") + || lower.contains("key") + { + return format!("{key}="); + } + } + pair.to_owned() + }) + .collect(); + format!("{path}?{}", redacted_query.join("&")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redacts_sensitive_query_parameters() { + let path = "/movie/stream.mkv?user=alice&password=secret123&token=abc&run=test-run"; + let redacted = redact_path_and_query(path); + assert!(redacted.contains("user=alice")); + assert!(redacted.contains("password=")); + assert!(redacted.contains("token=")); + assert!(redacted.contains("run=test-run")); + } +} diff --git a/tools/tuliprox-testkit/src/origin_transport.rs b/tools/tuliprox-testkit/src/origin_transport.rs new file mode 100644 index 000000000..a176d6d63 --- /dev/null +++ b/tools/tuliprox-testkit/src/origin_transport.rs @@ -0,0 +1,484 @@ +use crate::origin_events::{redact_path_and_query, BodyCloseReason, OriginEvent, OriginEventKind, OriginStats}; +use pin_project_lite::pin_project; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + net::SocketAddr, + pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, + task::{Context, Poll}, + time::Instant, +}; +use tokio::{ + io::{AsyncRead, AsyncWrite, ReadBuf}, + sync::{oneshot, Mutex}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OriginLimitMode { + ObserveOnly, + RejectNew, + EvictOldest, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OriginPolicy { + pub account_limit: Option, + pub limit_mode: OriginLimitMode, +} + +impl Default for OriginPolicy { + fn default() -> Self { Self { account_limit: None, limit_mode: OriginLimitMode::ObserveOnly } } +} + +#[derive(Clone, Debug)] +pub struct OriginConnectionMeta { + pub conn_id: u64, + pub remote_addr: SocketAddr, +} + +struct ActiveConnection { + conn_id: u64, + #[allow(dead_code)] + remote_addr: SocketAddr, + connected_at: Instant, + active_body_req_id: Option, +} + +#[derive(Clone)] +pub struct OriginTracker { + run_id: String, + policy: Arc>, + events: Arc>>, + stats: Arc>, + active_conns: Arc>>, + eviction_channels: Arc>>>, + seq_counter: Arc, + conn_counter: Arc, + req_counter: Arc, +} + +impl OriginTracker { + #[must_use] + pub fn new(run_id: impl Into, policy: OriginPolicy) -> Self { + Self { + run_id: run_id.into(), + policy: Arc::new(Mutex::new(policy)), + events: Arc::new(Mutex::new(Vec::new())), + stats: Arc::new(Mutex::new(OriginStats::default())), + active_conns: Arc::new(Mutex::new(HashMap::new())), + eviction_channels: Arc::new(Mutex::new(HashMap::new())), + seq_counter: Arc::new(AtomicU64::new(0)), + conn_counter: Arc::new(AtomicU64::new(0)), + req_counter: Arc::new(AtomicU64::new(0)), + } + } + + fn next_seq(&self) -> u64 { self.seq_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1) } + + pub async fn set_policy(&self, policy: OriginPolicy) { *self.policy.lock().await = policy; } + + pub async fn policy(&self) -> OriginPolicy { self.policy.lock().await.clone() } + + pub async fn on_tcp_accepted(&self, remote_addr: SocketAddr) -> u64 { + let conn_id = self.conn_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_tcp_connections = stats.active_tcp_connections.saturating_add(1); + stats.total_tcp_connections = stats.total_tcp_connections.saturating_add(1); + if stats.active_tcp_connections > stats.max_active_tcp_connections { + stats.max_active_tcp_connections = stats.active_tcp_connections; + } + drop(stats); + + self.active_conns.lock().await.insert( + conn_id, + ActiveConnection { conn_id, remote_addr, connected_at: Instant::now(), active_body_req_id: None }, + ); + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::TcpAccepted { conn_id, remote_addr: remote_addr.to_string() }, + )); + + conn_id + } + + pub async fn on_request_started( + &self, + conn_id: u64, + method: &str, + path_and_query: &str, + range: Option<&str>, + user_agent: Option<&str>, + account: Option<&str>, + ) -> Result { + let req_id = self.req_counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.total_requests = stats.total_requests.saturating_add(1); + let active_bodies = stats.active_body_connections; + drop(stats); + + let policy = self.policy.lock().await.clone(); + if let Some(limit) = policy.account_limit { + if active_bodies >= limit { + match policy.limit_mode { + OriginLimitMode::ObserveOnly => {} + OriginLimitMode::RejectNew => { + let reject_seq = self.next_seq(); + self.events.lock().await.push(OriginEvent::new( + reject_seq, + &self.run_id, + OriginEventKind::LimitRejected { + conn_id, + request_id: req_id, + limit, + current_active: active_bodies, + }, + )); + return Err((req_id, 429)); + } + OriginLimitMode::EvictOldest => { + self.evict_oldest_body(conn_id).await; + } + } + } + } + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::RequestStarted { + conn_id, + request_id: req_id, + method: method.to_owned(), + path: redact_path_and_query(path_and_query), + range: range.map(str::to_owned), + user_agent: user_agent.map(str::to_owned), + account: account.map(str::to_owned), + }, + )); + + Ok(req_id) + } + + async fn evict_oldest_body(&self, triggering_conn_id: u64) { + let conns = self.active_conns.lock().await; + let oldest_active = conns + .values() + .filter(|c| c.active_body_req_id.is_some() && c.conn_id != triggering_conn_id) + .min_by_key(|c| c.connected_at); + + if let Some(oldest) = oldest_active { + let evicted_conn_id = oldest.conn_id; + let evicted_req_id = oldest.active_body_req_id.unwrap_or(0); + drop(conns); + + if let Some(sender) = self.eviction_channels.lock().await.remove(&evicted_conn_id) { + let _ = sender.send(()); + } + + let seq = self.next_seq(); + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::EvictionTriggered { + evicted_conn_id, + evicted_request_id: evicted_req_id, + triggering_conn_id, + }, + )); + } + } + + pub async fn on_body_started( + &self, + conn_id: u64, + req_id: u64, + content_length: Option, + ) -> oneshot::Receiver<()> { + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_body_connections = stats.active_body_connections.saturating_add(1); + if stats.active_body_connections > stats.max_active_body_connections { + stats.max_active_body_connections = stats.active_body_connections; + } + drop(stats); + + if let Some(conn) = self.active_conns.lock().await.get_mut(&conn_id) { + conn.active_body_req_id = Some(req_id); + } + + let (tx, rx) = oneshot::channel(); + self.eviction_channels.lock().await.insert(conn_id, tx); + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::BodyStarted { conn_id, request_id: req_id, content_length }, + )); + + rx + } + + pub async fn on_body_closed( + &self, + conn_id: u64, + req_id: u64, + bytes_emitted: u64, + duration_ms: u64, + reason: BodyCloseReason, + ) { + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_body_connections = stats.active_body_connections.saturating_sub(1); + stats.total_bytes_emitted = stats.total_bytes_emitted.saturating_add(bytes_emitted); + drop(stats); + + self.eviction_channels.lock().await.remove(&conn_id); + if let Some(conn) = self.active_conns.lock().await.get_mut(&conn_id) { + if conn.active_body_req_id == Some(req_id) { + conn.active_body_req_id = None; + } + } + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::BodyClosed { conn_id, request_id: req_id, bytes_emitted, duration_ms, reason }, + )); + } + + pub async fn on_tcp_closed( + &self, + conn_id: u64, + duration_ms: u64, + bytes_read: u64, + bytes_written: u64, + reason: &str, + ) { + let seq = self.next_seq(); + + let mut stats = self.stats.lock().await; + stats.active_tcp_connections = stats.active_tcp_connections.saturating_sub(1); + drop(stats); + + self.active_conns.lock().await.remove(&conn_id); + self.eviction_channels.lock().await.remove(&conn_id); + + self.events.lock().await.push(OriginEvent::new( + seq, + &self.run_id, + OriginEventKind::TcpClosed { conn_id, duration_ms, bytes_read, bytes_written, reason: reason.to_owned() }, + )); + } + + pub async fn stats(&self) -> OriginStats { self.stats.lock().await.clone() } + + pub async fn events(&self) -> Vec { self.events.lock().await.clone() } + + pub async fn reset(&self) { + *self.stats.lock().await = OriginStats::default(); + self.events.lock().await.clear(); + self.active_conns.lock().await.clear(); + self.eviction_channels.lock().await.clear(); + self.seq_counter.store(0, Ordering::Relaxed); + self.conn_counter.store(0, Ordering::Relaxed); + self.req_counter.store(0, Ordering::Relaxed); + } +} + +struct TcpDropGuard { + conn_id: u64, + tracker: OriginTracker, + start_time: Instant, + bytes_read: Arc, + bytes_written: Arc, + closed: Arc, +} + +impl Drop for TcpDropGuard { + fn drop(&mut self) { + if !self.closed.swap(true, Ordering::SeqCst) { + let conn_id = self.conn_id; + let tracker = self.tracker.clone(); + let duration_ms = u64::try_from(self.start_time.elapsed().as_millis()).unwrap_or(u64::MAX); + let bytes_read = self.bytes_read.load(Ordering::Relaxed); + let bytes_written = self.bytes_written.load(Ordering::Relaxed); + tokio::spawn(async move { + tracker.on_tcp_closed(conn_id, duration_ms, bytes_read, bytes_written, "io_dropped").await; + }); + } + } +} + +pin_project! { + pub struct TrackedIo { + #[pin] + inner: T, + bytes_read: Arc, + bytes_written: Arc, + _guard: TcpDropGuard, + } +} + +impl TrackedIo { + pub fn new(inner: T, conn_id: u64, tracker: OriginTracker) -> (Self, Arc) { + let closed = Arc::new(AtomicBool::new(false)); + let bytes_read = Arc::new(AtomicU64::new(0)); + let bytes_written = Arc::new(AtomicU64::new(0)); + let guard = TcpDropGuard { + conn_id, + tracker, + start_time: Instant::now(), + bytes_read: bytes_read.clone(), + bytes_written: bytes_written.clone(), + closed: closed.clone(), + }; + (Self { inner, bytes_read, bytes_written, _guard: guard }, closed) + } +} + +impl AsyncRead for TrackedIo { + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let this = self.project(); + let prev_len = buf.filled().len(); + let poll_res = this.inner.poll_read(cx, buf); + if let Poll::Ready(Ok(())) = &poll_res { + let newly_read = buf.filled().len().saturating_sub(prev_len) as u64; + this.bytes_read.fetch_add(newly_read, Ordering::Relaxed); + } + poll_res + } +} + +impl AsyncWrite for TrackedIo { + fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + let this = self.project(); + let poll_res = this.inner.poll_write(cx, buf); + if let Poll::Ready(Ok(written)) = &poll_res { + this.bytes_written.fetch_add(*written as u64, Ordering::Relaxed); + } + poll_res + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_shutdown(cx) + } +} + +pub async fn serve_tracked_media_listener( + listener: tokio::net::TcpListener, + app: axum::Router, + tracker: OriginTracker, +) -> Result<(), std::io::Error> { + loop { + let (tcp_stream, remote_addr) = listener.accept().await?; + let conn_id = tracker.on_tcp_accepted(remote_addr).await; + let (tracked_io, _closed) = TrackedIo::new(tcp_stream, conn_id, tracker.clone()); + let io = hyper_util::rt::TokioIo::new(tracked_io); + let app = app.clone(); + let conn_meta = OriginConnectionMeta { conn_id, remote_addr }; + + tokio::spawn(async move { + let service = hyper::service::service_fn(move |req: hyper::Request| { + let mut req = req.map(axum::body::Body::new); + req.extensions_mut().insert(conn_meta.clone()); + let mut app = app.clone(); + async move { tower_service::Service::call(&mut app, req).await } + }); + + let builder = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); + let _ = builder.serve_connection(io, service).await; + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{IpAddr, Ipv4Addr}; + + #[tokio::test] + async fn tracking_records_tcp_lifecycle_and_peak_counters() { + let tracker = OriginTracker::new("test-run", OriginPolicy::default()); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); + let conn_id = tracker.on_tcp_accepted(addr).await; + assert_eq!(conn_id, 1); + + let stats = tracker.stats().await; + assert_eq!(stats.active_tcp_connections, 1); + assert_eq!(stats.max_active_tcp_connections, 1); + + let req_id = tracker + .on_request_started(conn_id, "GET", "/live/17.ts", None, Some("test/1"), None) + .await + .expect("request admitted"); + assert_eq!(req_id, 1); + + let _evict_rx = tracker.on_body_started(conn_id, req_id, None).await; + let stats = tracker.stats().await; + assert_eq!(stats.active_body_connections, 1); + assert_eq!(stats.max_active_body_connections, 1); + + tracker.on_body_closed(conn_id, req_id, 1024, 50, BodyCloseReason::Completed).await; + let stats = tracker.stats().await; + assert_eq!(stats.active_body_connections, 0); + assert_eq!(stats.max_active_body_connections, 1); + + tracker.on_tcp_closed(conn_id, 100, 200, 1024, "clean_close").await; + let stats = tracker.stats().await; + assert_eq!(stats.active_tcp_connections, 0); + assert_eq!(stats.max_active_tcp_connections, 1); + + let events = tracker.events().await; + assert_eq!(events.len(), 5); + } + + #[tokio::test] + async fn policy_rejects_new_when_account_limit_reached() { + let policy = OriginPolicy { account_limit: Some(1), limit_mode: OriginLimitMode::RejectNew }; + let tracker = OriginTracker::new("test-run", policy); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); + + let conn1 = tracker.on_tcp_accepted(addr).await; + let req1 = tracker.on_request_started(conn1, "GET", "/live/1.ts", None, None, None).await.unwrap(); + let _rx1 = tracker.on_body_started(conn1, req1, None).await; + + let conn2 = tracker.on_tcp_accepted(addr).await; + let req2_res = tracker.on_request_started(conn2, "GET", "/live/2.ts", None, None, None).await; + assert_eq!(req2_res, Err((2, 429))); + } + + #[tokio::test] + async fn policy_evicts_oldest_when_account_limit_reached() { + let policy = OriginPolicy { account_limit: Some(1), limit_mode: OriginLimitMode::EvictOldest }; + let tracker = OriginTracker::new("test-run", policy); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); + + let conn1 = tracker.on_tcp_accepted(addr).await; + let req1 = tracker.on_request_started(conn1, "GET", "/live/1.ts", None, None, None).await.unwrap(); + let mut rx1 = tracker.on_body_started(conn1, req1, None).await; + + let conn2 = tracker.on_tcp_accepted(addr).await; + let req2 = tracker.on_request_started(conn2, "GET", "/live/2.ts", None, None, None).await.unwrap(); + assert_eq!(req2, 2); + + // conn1 should have received eviction signal + assert!(rx1.try_recv().is_ok()); + } +} diff --git a/tools/tuliprox-testkit/src/policy.rs b/tools/tuliprox-testkit/src/policy.rs new file mode 100644 index 000000000..fc9d05be5 --- /dev/null +++ b/tools/tuliprox-testkit/src/policy.rs @@ -0,0 +1,271 @@ +use crate::{ + config::{GraceContract, PolicyContract}, + oracle::{AdmissionStrategy, GraceMode}, + TestkitError, +}; +use serde_json::Value; +use std::collections::HashMap; + +/// Validate the persisted fixture configuration before any playback is started. +/// +/// The v1 configuration endpoint intentionally returns configuration files, rather +/// than the atomically loaded runtime configuration. The controller therefore +/// uses this only for isolated fixtures it started itself; a remote instance needs +/// runtime evidence and is reported as inconclusive by the caller. +pub fn validate_fixture_policy(contract: &PolicyContract, config: &Value) -> Result<(), TestkitError> { + let configured_access_control = required_bool(config, "/config/user_access_control")?; + if configured_access_control != contract.user_access_control { + return mismatch("user_access_control", contract.user_access_control, configured_access_control); + } + + let stream = required_object(config, "/config/reverse_proxy/stream")?; + let configured_strategies = stream + .get("admission_strategies") + .map(|strategies| { + serde_json::from_value::>(strategies.clone()) + .map_err(|error| TestkitError::Protocol(format!("invalid fixture admission_strategies: {error}"))) + }) + .transpose()?; + if configured_strategies != contract.admission_strategies { + return Err(TestkitError::Configuration(format!( + "fixture admission_strategies mismatch: scenario {:?}, fixture {:?}", + contract.admission_strategies, configured_strategies + ))); + } + + if let Some(grace) = &contract.grace { + validate_grace(grace, stream, &contract.effective_admission_strategies())?; + } + + let configured_users = index_users(config)?; + for (username, expected) in &contract.users { + let configured = configured_users + .get(username.as_str()) + .ok_or_else(|| TestkitError::Configuration(format!("fixture policy does not contain user {username}")))?; + let max_connections = usize::try_from(required_u64(configured, "max_connections")?).map_err(|_| { + TestkitError::Protocol("fixture max_connections cannot be represented on this platform".to_owned()) + })?; + let soft_connections = usize::try_from(required_u64(configured, "soft_connections")?).map_err(|_| { + TestkitError::Protocol("fixture soft_connections cannot be represented on this platform".to_owned()) + })?; + if max_connections != expected.max_connections || soft_connections != expected.soft_connections { + return Err(TestkitError::Configuration(format!( + "fixture policy mismatch for user {username}: scenario max/soft {}/{}, fixture {}/{}", + expected.max_connections, expected.soft_connections, max_connections, soft_connections + ))); + } + } + + if let Some(expected_limit) = contract.provider_max_connections { + validate_provider_limit(expected_limit.into(), config)?; + } + + Ok(()) +} + +fn validate_provider_limit(expected_limit: usize, config: &Value) -> Result<(), TestkitError> { + let inputs = config + .pointer("/sources/inputs") + .and_then(Value::as_array) + .ok_or_else(|| TestkitError::Protocol("fixture configuration omits sources.inputs".to_owned()))?; + + let matching_input = inputs + .iter() + .find(|input| { + input.get("name").and_then(Value::as_str) == Some("testkit-origin") + || input.get("max_connections").is_some() + }) + .ok_or_else(|| { + TestkitError::Configuration( + "fixture sources do not contain a provider input with max_connections".to_owned(), + ) + })?; + + let configured_limit = matching_input + .get("max_connections") + .and_then(Value::as_u64) + .ok_or_else(|| TestkitError::Configuration("fixture provider input omits max_connections".to_owned()))?; + + let configured_limit = usize::try_from(configured_limit).map_err(|_| { + TestkitError::Protocol("fixture provider max_connections cannot be represented on this platform".to_owned()) + })?; + + if configured_limit != expected_limit { + return Err(TestkitError::Configuration(format!( + "fixture provider max_connections mismatch: scenario {expected_limit}, fixture {configured_limit}" + ))); + } + Ok(()) +} + +fn validate_grace( + expected: &GraceContract, + stream: &serde_json::Map, + strategies: &[AdmissionStrategy], +) -> Result<(), TestkitError> { + const DEFAULT_GRACE_PERIOD_MILLIS: u64 = 2000; + const DEFAULT_GRACE_HOLD_STREAM: bool = true; + + let timeout = match stream.get("grace_period_millis") { + Some(value) => value.as_u64().ok_or_else(|| { + TestkitError::Protocol( + "fixture configuration omits integer config.reverse_proxy.stream.grace_period_millis".to_owned(), + ) + })?, + None => DEFAULT_GRACE_PERIOD_MILLIS, + }; + if timeout != expected.timeout_millis { + return Err(TestkitError::Configuration(format!( + "fixture grace_period_millis mismatch: scenario {}, fixture {timeout}", + expected.timeout_millis + ))); + } + let hold = match stream.get("grace_period_hold_stream") { + Some(value) => value.as_bool().ok_or_else(|| { + TestkitError::Protocol( + "fixture configuration omits boolean config.reverse_proxy.stream.grace_period_hold_stream".to_owned(), + ) + })?, + None => DEFAULT_GRACE_HOLD_STREAM, + }; + let expected_hold = expected.mode == GraceMode::HoldStream; + if hold != expected_hold { + return mismatch("grace_period_hold_stream", expected_hold, hold); + } + // A grace mode without its corresponding strategy would be silently ignored + // by the SUT after capacity is exhausted. + let grace_strategy = match expected.mode { + GraceMode::Instant => "grace_instant_stream", + GraceMode::HoldStream => "grace_hold_stream", + }; + let encoded = serde_json::to_value(strategies) + .map_err(|error| TestkitError::Protocol(format!("cannot encode admission strategies: {error}")))?; + if !encoded.as_array().is_some_and(|values| values.iter().any(|value| value == grace_strategy)) { + return Err(TestkitError::Configuration(format!( + "fixture policy grace mode requires {grace_strategy} admission strategy" + ))); + } + Ok(()) +} + +fn index_users(config: &Value) -> Result, TestkitError> { + let users = config + .pointer("/api_proxy/user") + .and_then(Value::as_array) + .ok_or_else(|| TestkitError::Protocol("fixture configuration omits api_proxy.user".to_owned()))?; + let mut indexed = HashMap::new(); + for target in users { + let credentials = target + .get("credentials") + .and_then(Value::as_array) + .ok_or_else(|| TestkitError::Protocol("fixture api_proxy user entry omits credentials".to_owned()))?; + for user in credentials { + let username = user + .get("username") + .and_then(Value::as_str) + .ok_or_else(|| TestkitError::Protocol("fixture user omits username".to_owned()))?; + if indexed.insert(username, user).is_some() { + return Err(TestkitError::Configuration(format!("fixture has duplicate username {username}"))); + } + } + } + Ok(indexed) +} + +fn required_object<'a>(value: &'a Value, pointer: &str) -> Result<&'a serde_json::Map, TestkitError> { + value + .pointer(pointer) + .and_then(Value::as_object) + .ok_or_else(|| TestkitError::Protocol(format!("fixture configuration omits object {pointer}"))) +} + +fn required_bool(value: &Value, pointer: &str) -> Result { + value + .pointer(pointer) + .and_then(Value::as_bool) + .ok_or_else(|| TestkitError::Protocol(format!("fixture configuration omits boolean {pointer}"))) +} + +fn required_u64(value: &Value, key: &str) -> Result { + value + .get(key) + .and_then(Value::as_u64) + .ok_or_else(|| TestkitError::Protocol(format!("fixture user omits integer {key}"))) +} + +fn mismatch(field: &str, expected: T, actual: T) -> Result<(), TestkitError> { + Err(TestkitError::Configuration(format!("fixture {field} mismatch: scenario {expected}, fixture {actual}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::UserPolicy; + use serde_json::json; + + fn contract() -> PolicyContract { + PolicyContract { + user_access_control: true, + users: HashMap::from([("alice".to_owned(), UserPolicy { max_connections: 2, soft_connections: 1 })]), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpLatest]), + recent_eviction_reentry_ttl_ms: None, + grace: None, + provider_max_connections: None, + expected_provider_slots: None, + } + } + + fn fixture() -> Value { + json!({ + "config": {"user_access_control": true, "reverse_proxy": {"stream": {"admission_strategies": ["evict_user_same_ip_latest"]}}}, + "api_proxy": {"user": [{"credentials": [{"username": "alice", "max_connections": 2, "soft_connections": 1}]}]} + }) + } + + #[test] + fn accepts_matching_fixture_policy() { + assert!(validate_fixture_policy(&contract(), &fixture()).is_ok()); + } + + #[test] + fn rejects_mismatched_user_limit() { + let mut config = fixture(); + config["api_proxy"]["user"][0]["credentials"][0]["max_connections"] = json!(3); + assert!(validate_fixture_policy(&contract(), &config).is_err()); + } + + #[test] + fn distinguishes_omitted_and_explicit_admission_strategies() { + let mut config = fixture(); + config["config"]["reverse_proxy"]["stream"].as_object_mut().unwrap().remove("admission_strategies"); + assert!(matches!(validate_fixture_policy(&contract(), &config), Err(TestkitError::Configuration(_)))); + + let mut omitted_contract = contract(); + omitted_contract.admission_strategies = None; + assert!(validate_fixture_policy(&omitted_contract, &config).is_ok()); + } + + #[test] + fn validates_provider_max_connections() { + let mut contract_with_provider = contract(); + contract_with_provider.provider_max_connections = Some(2); + + let mut config_matching = fixture(); + config_matching["sources"] = json!({ + "inputs": [{"name": "testkit-origin", "max_connections": 2}] + }); + assert!(validate_fixture_policy(&contract_with_provider, &config_matching).is_ok()); + + let mut config_mismatched = fixture(); + config_mismatched["sources"] = json!({ + "inputs": [{"name": "testkit-origin", "max_connections": 1}] + }); + assert!(validate_fixture_policy(&contract_with_provider, &config_mismatched).is_err()); + + let mut config_missing = fixture(); + config_missing["sources"] = json!({ + "inputs": [{"name": "other-origin"}] + }); + assert!(validate_fixture_policy(&contract_with_provider, &config_missing).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/protocol.rs b/tools/tuliprox-testkit/src/protocol.rs new file mode 100644 index 000000000..e45d7747a --- /dev/null +++ b/tools/tuliprox-testkit/src/protocol.rs @@ -0,0 +1,235 @@ +use crate::TestkitError; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +pub const SCHEMA_VERSION: u16 = 1; + +macro_rules! id_type { + ($name:ident) => { + #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[serde(transparent)] + pub struct $name(pub String); + + impl $name { + #[must_use] + pub fn new(value: impl Into) -> Self { Self(value.into()) } + } + }; +} + +id_type!(RunId); +id_type!(AgentId); +id_type!(ActorId); +id_type!(PlaybackId); +id_type!(CommandId); +id_type!(OriginStreamId); + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Envelope { + pub schema_version: u16, + pub run_id: RunId, + pub run_generation: u64, + pub message_id: String, + pub agent_id: AgentId, + pub agent_boot_id: String, + pub source_sequence: u64, + pub caused_by_command_id: Option, + pub local_elapsed_nanos: u64, + pub payload: T, +} + +impl Envelope { + pub fn validate(&self) -> Result<(), TestkitError> { + if self.schema_version != SCHEMA_VERSION { + return Err(TestkitError::Protocol(format!("unsupported schema version {}", self.schema_version))); + } + if self.run_id.0.is_empty() || self.message_id.is_empty() || self.agent_id.0.is_empty() { + return Err(TestkitError::Protocol("empty envelope identity".to_owned())); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum Command { + ConfigureRun { + lease_millis: u64, + }, + StartPlayback { + playback_id: PlaybackId, + url: String, + #[serde(default)] + headers: std::collections::BTreeMap, + #[serde(default)] + expected_run_id: Option, + #[serde(default)] + expected_marker: Option, + #[serde(default)] + required_frames: Option, + }, + StopPlayback { + playback_id: PlaybackId, + }, + StopAll, + RenewRunLease { + lease_millis: u64, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RejectionReason { + CustomVideo(crate::custom_video::CustomVideoKind), + HttpStatus(u16), + ClosedBeforeFirstByte, + Other(String), +} + +impl std::fmt::Display for RejectionReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::CustomVideo(kind) => write!(f, "custom-video({kind:?})"), + Self::HttpStatus(status) => write!(f, "http-status({status})"), + Self::ClosedBeforeFirstByte => write!(f, "closed-before-first-byte"), + Self::Other(msg) => write!(f, "{msg}"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum PlaybackOutcome { + Streaming { frames: u64, bytes: u64 }, + AdmissionRejected { reason: RejectionReason }, + HttpError { status: u16 }, + TransportError { message: String }, + InvalidData { message: String }, + UnexpectedEof { frames: u64, bytes: u64 }, + IdleTimeout, + ExplicitStop, + InfrastructureError { message: String }, +} + +impl PlaybackOutcome { + #[must_use] + pub fn is_streaming(&self) -> bool { matches!(self, Self::Streaming { .. }) } + + #[must_use] + pub fn is_rejected(&self) -> bool { matches!(self, Self::AdmissionRejected { .. }) } +} + +impl std::fmt::Display for PlaybackOutcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Streaming { frames, bytes } => write!(f, "streaming({frames} frames, {bytes} bytes)"), + Self::AdmissionRejected { reason } => write!(f, "rejected({reason})"), + Self::HttpError { status } => write!(f, "http_error({status})"), + Self::TransportError { message } => write!(f, "transport_error({message})"), + Self::InvalidData { message } => write!(f, "invalid_data({message})"), + Self::UnexpectedEof { frames, bytes } => write!(f, "unexpected_eof({frames} frames, {bytes} bytes)"), + Self::IdleTimeout => write!(f, "idle_timeout"), + Self::ExplicitStop => write!(f, "explicit_stop"), + Self::InfrastructureError { message } => write!(f, "infrastructure_error({message})"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum PlaybackEvent { + CommandAccepted { + command_id: CommandId, + }, + HeadersReceived { + playback_id: PlaybackId, + status: u16, + }, + FirstValidFrame { + playback_id: PlaybackId, + sequence: u64, + }, + Progress { + playback_id: PlaybackId, + frames: u64, + bytes: u64, + }, + Terminal { + playback_id: PlaybackId, + outcome: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + typed_outcome: Option, + }, + LeaseExpired, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum AgentMessage { + Hello { hostname: String, supported_protocols: Vec, maximum_listeners: u32 }, + Ready, + Event { event: PlaybackEvent }, + Heartbeat, +} + +#[derive(Debug, Clone)] +pub struct RunLease { + generation: u64, + expires_at: tokio::time::Instant, +} + +impl RunLease { + #[must_use] + pub fn new(generation: u64, duration: Duration) -> Self { + Self { generation, expires_at: tokio::time::Instant::now() + duration } + } + + pub fn renew(&mut self, generation: u64, duration: Duration) -> Result<(), TestkitError> { + if generation != self.generation { + return Err(TestkitError::Protocol("stale run generation".to_owned())); + } + self.expires_at = tokio::time::Instant::now() + duration; + Ok(()) + } + + #[must_use] + pub fn expired(&self) -> bool { tokio::time::Instant::now() >= self.expires_at } + + #[must_use] + pub const fn deadline(&self) -> tokio::time::Instant { self.expires_at } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_unknown_schema() { + let envelope = Envelope { + schema_version: 2, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "message".to_owned(), + agent_id: AgentId::new("agent"), + agent_boot_id: "boot".to_owned(), + source_sequence: 0, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: Command::StopAll, + }; + assert!(envelope.validate().is_err()); + } + + #[tokio::test(start_paused = true)] + async fn lease_expiry_and_renewal_use_monotonic_time() { + let mut lease = RunLease::new(1, Duration::from_secs(5)); + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!lease.expired()); + lease.renew(1, Duration::from_secs(5)).unwrap(); + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!lease.expired()); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(lease.expired()); + } +} diff --git a/tools/tuliprox-testkit/src/report.rs b/tools/tuliprox-testkit/src/report.rs new file mode 100644 index 000000000..da5cae0b2 --- /dev/null +++ b/tools/tuliprox-testkit/src/report.rs @@ -0,0 +1,146 @@ +use crate::TestkitError; +use serde::Serialize; +use std::{fs, path::Path}; + +#[derive(Debug, Serialize, serde::Deserialize)] +pub struct RunReport<'a> { + pub schema_version: u16, + pub scenario: &'a str, + pub outcome: &'a str, + pub policy_hash: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub run_id: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_kind: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_message: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub exit_code: Option, + pub events: Vec>, +} + +#[derive(Debug, Serialize, serde::Deserialize)] +pub struct ReportEvent<'a> { + pub source: &'a str, + pub playback_id: &'a str, + pub event: &'a str, + pub detail: &'a str, +} + +impl RunReport<'_> { + pub fn write_to(&self, directory: &Path) -> Result<(), TestkitError> { + fs::create_dir_all(directory)?; + let json = serde_json::to_vec_pretty(self) + .map_err(|error| TestkitError::Protocol(format!("cannot serialize report: {error}")))?; + fs::write(directory.join("run.json"), json)?; + fs::write(directory.join("events.ndjson"), self.events_ndjson()?)?; + let run_id_line = self.run_id.map_or_else(String::new, |id| format!("run_id: {id}\n")); + let summary = format!( + "scenario: {}\noutcome: {}\npolicy: {}\n{}events: {}\n", + self.scenario, + self.outcome, + self.policy_hash, + run_id_line, + self.events.len() + ); + fs::write(directory.join("summary.txt"), summary)?; + fs::write(directory.join("junit.xml"), self.junit_xml())?; + Ok(()) + } + + fn junit_xml(&self) -> String { + let failed = usize::from(self.outcome != "passed"); + let name = xml_escape(self.scenario); + if failed == 0 { + format!( + "\n\n" + ) + } else { + format!( + "\n\n", + xml_escape(self.outcome) + ) + } + } + + fn events_ndjson(&self) -> Result { + let mut lines = String::new(); + for event in &self.events { + let encoded = serde_json::to_string(event) + .map_err(|error| TestkitError::Protocol(format!("cannot serialize report event: {error}")))?; + lines.push_str(&encoded); + lines.push('\n'); + } + Ok(lines) + } +} + +fn xml_escape(value: &str) -> String { + value.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn junit_marks_failed_runs() { + let report = RunReport { + schema_version: 1, + scenario: "a&b", + outcome: "failed", + policy_hash: "hash", + run_id: None, + error_kind: None, + error_message: None, + exit_code: None, + events: Vec::new(), + }; + assert!(report.junit_xml().contains("failures=\"1\"")); + assert!(report.junit_xml().contains("a&b")); + } + + #[test] + fn events_are_emitted_as_one_json_object_per_line() { + let report = RunReport { + schema_version: 1, + scenario: "scenario", + outcome: "passed", + policy_hash: "hash", + run_id: None, + error_kind: None, + error_message: None, + exit_code: None, + events: vec![ReportEvent { + source: "controller", + playback_id: "playback-a", + event: "started", + detail: "ok", + }], + }; + let lines = report.events_ndjson().unwrap(); + assert_eq!(lines.lines().count(), 1); + assert_eq!(serde_json::from_str::(lines.trim()).unwrap()["playback_id"], "playback-a"); + } + + #[test] + fn report_serializes_and_deserializes_with_run_id_and_error() { + let report = RunReport { + schema_version: 1, + scenario: "scenario_fail", + outcome: "failed", + policy_hash: "pre-start", + run_id: Some("test-run-123"), + error_kind: Some("bootstrap_error"), + error_message: Some("address already in use"), + exit_code: Some(4), + events: Vec::new(), + }; + let json = serde_json::to_string(&report).unwrap(); + let value: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(value["run_id"], "test-run-123"); + assert_eq!(value["error_kind"], "bootstrap_error"); + assert_eq!(value["error_message"], "address already in use"); + assert_eq!(value["exit_code"], 4); + } +} diff --git a/tools/tuliprox-testkit/src/scheduler.rs b/tools/tuliprox-testkit/src/scheduler.rs new file mode 100644 index 000000000..1169ce0b0 --- /dev/null +++ b/tools/tuliprox-testkit/src/scheduler.rs @@ -0,0 +1,85 @@ +use crate::TestkitError; +use std::{collections::HashMap, time::Duration}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct EventKey { + pub command_id: String, + pub event: String, +} + +#[derive(Debug, Default)] +pub struct Timeline { + anchors: HashMap, +} + +impl Timeline { + pub fn record(&mut self, key: EventKey) -> Result<(), TestkitError> { + if self.anchors.contains_key(&key) { + return Err(TestkitError::Protocol(format!("duplicate causal event {}:{}", key.command_id, key.event))); + } + self.anchors.insert(key, tokio::time::Instant::now()); + Ok(()) + } + + pub fn deadline(&self, key: &EventKey, within: Duration) -> Result { + self.anchors.get(key).copied().map(|anchor| anchor + within).ok_or_else(|| { + TestkitError::Configuration(format!("assertion anchor {}:{} was never armed", key.command_id, key.event)) + }) + } + + pub fn require_before( + &self, + key: &EventKey, + within: Duration, + observed_at: tokio::time::Instant, + ) -> Result<(), TestkitError> { + if observed_at > self.deadline(key, within)? { + return Err(TestkitError::Protocol(format!( + "event exceeded deadline anchored at {}:{}", + key.command_id, key.event + ))); + } + Ok(()) + } +} + +#[derive(Debug)] +pub struct ContinuousProgress { + max_gap: Duration, + last_frame: Option, +} + +impl ContinuousProgress { + #[must_use] + pub fn new(max_gap: Duration) -> Self { Self { max_gap, last_frame: None } } + + pub fn frame(&mut self, now: tokio::time::Instant) -> Result<(), TestkitError> { + if self.last_frame.is_some_and(|previous| now.duration_since(previous) > self.max_gap) { + return Err(TestkitError::Protocol("stream progress gap exceeded assertion bound".to_owned())); + } + self.last_frame = Some(now); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn deadline_remains_anchored_when_later_events_arrive() { + let mut timeline = Timeline::default(); + let key = EventKey { command_id: "start-b".to_owned(), event: "sent".to_owned() }; + timeline.record(key.clone()).unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(timeline.require_before(&key, Duration::from_secs(1), tokio::time::Instant::now()).is_err()); + } + + #[tokio::test(start_paused = true)] + async fn continuous_progress_rejects_long_gaps() { + let mut progress = ContinuousProgress::new(Duration::from_secs(1)); + progress.frame(tokio::time::Instant::now()).unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(progress.frame(tokio::time::Instant::now()).is_err()); + } +} diff --git a/tools/tuliprox-testkit/src/secret.rs b/tools/tuliprox-testkit/src/secret.rs new file mode 100644 index 000000000..77e532f34 --- /dev/null +++ b/tools/tuliprox-testkit/src/secret.rs @@ -0,0 +1,20 @@ +use crate::TestkitError; + +pub fn environment(name: &str) -> Result { + if name.trim().is_empty() { + return Err(TestkitError::Configuration("secret environment variable name is empty".to_owned())); + } + std::env::var(name) + .map_err(|_| TestkitError::Configuration(format!("required secret environment variable {name} is not set"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_variable_does_not_expose_a_value() { + let error = environment("TULIPROX_TESTKIT_MISSING_SECRET").unwrap_err().to_string(); + assert!(error.contains("TULIPROX_TESTKIT_MISSING_SECRET")); + } +} diff --git a/tools/tuliprox-testkit/src/shared.rs b/tools/tuliprox-testkit/src/shared.rs new file mode 100644 index 000000000..5f8f43fce --- /dev/null +++ b/tools/tuliprox-testkit/src/shared.rs @@ -0,0 +1,53 @@ +use crate::TestkitError; +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Default)] +pub struct SharedStreamModel { + subscribers: HashMap>, +} + +impl SharedStreamModel { + /// Returns true only when this subscriber causes a new origin connection. + pub fn subscribe(&mut self, marker: u32, playback_id: String) -> Result { + let subscribers = self.subscribers.entry(marker).or_default(); + if !subscribers.insert(playback_id) { + return Err(TestkitError::Protocol("shared-stream subscriber was registered twice".to_owned())); + } + Ok(subscribers.len() == 1) + } + + /// Returns true only when the last subscriber closes the origin connection. + pub fn unsubscribe(&mut self, marker: u32, playback_id: &str) -> Result { + let subscribers = self + .subscribers + .get_mut(&marker) + .ok_or_else(|| TestkitError::Configuration(format!("shared stream marker {marker} is not active")))?; + if !subscribers.remove(playback_id) { + return Err(TestkitError::Configuration(format!( + "playback {playback_id} is not a shared-stream subscriber" + ))); + } + let last = subscribers.is_empty(); + if last { + self.subscribers.remove(&marker); + } + Ok(last) + } + + #[must_use] + pub fn subscriber_count(&self, marker: u32) -> usize { self.subscribers.get(&marker).map_or(0, HashSet::len) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_first_and_last_subscriber_change_origin_lifetime() { + let mut model = SharedStreamModel::default(); + assert!(model.subscribe(17, "a".to_owned()).unwrap()); + assert!(!model.subscribe(17, "b".to_owned()).unwrap()); + assert!(!model.unsubscribe(17, "a").unwrap()); + assert!(model.unsubscribe(17, "b").unwrap()); + } +} diff --git a/tools/tuliprox-testkit/src/transport.rs b/tools/tuliprox-testkit/src/transport.rs new file mode 100644 index 000000000..89d9dd4af --- /dev/null +++ b/tools/tuliprox-testkit/src/transport.rs @@ -0,0 +1,493 @@ +use crate::{ + protocol::{AgentId, AgentMessage, Command, Envelope, PlaybackEvent, RunId}, + TestkitError, +}; +use axum::{ + extract::{ + ws::{Message, WebSocket}, + State, WebSocketUpgrade, + }, + response::{IntoResponse, Response}, +}; +use futures::{SinkExt, StreamExt}; +use std::{ + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + sync::{mpsc, Mutex}, + time::Instant, +}; +use tokio_tungstenite::{connect_async, tungstenite::protocol::Message as ClientMessage}; + +pub type ControllerState = Arc; + +/// One connected agent registration. +/// +/// `token` identifies the exact websocket connection that registered the agent. +/// A replacement connection for the same `AgentId` overwrites the registration +/// with a new token, so the previous connection's cleanup cannot delete it. +struct RegisteredAgent { + token: u64, + sender: mpsc::Sender>, +} + +pub struct SharedControllerState { + run_id: RunId, + generation: u64, + agents: Mutex>, + ready_agents: Mutex>, + last_seen: Mutex>, + events: Mutex>>, + next_controller_sequence: AtomicU64, + next_connection_token: AtomicU64, +} + +impl SharedControllerState { + #[must_use] + pub fn new(run_id: RunId, generation: u64) -> Self { + Self { + run_id, + generation, + agents: Mutex::new(HashMap::new()), + ready_agents: Mutex::new(HashSet::new()), + last_seen: Mutex::new(HashMap::new()), + events: Mutex::new(Vec::new()), + next_controller_sequence: AtomicU64::new(1), + next_connection_token: AtomicU64::new(1), + } + } + + #[must_use] + pub fn next_command_sequence(&self) -> u64 { self.next_controller_sequence.fetch_add(1, Ordering::Relaxed) } + + fn next_connection_token(&self) -> u64 { self.next_connection_token.fetch_add(1, Ordering::Relaxed) } + + async fn register_agent(&self, agent_id: AgentId, token: u64, sender: mpsc::Sender>) { + let mut agents = self.agents.lock().await; + agents.insert(agent_id.clone(), RegisteredAgent { token, sender }); + self.last_seen.lock().await.insert(agent_id, Instant::now()); + } + + async fn unregister_agent(&self, agent_id: &AgentId, token: u64) { + let mut agents = self.agents.lock().await; + if agents.get(agent_id).is_some_and(|registered| registered.token == token) { + agents.remove(agent_id); + self.ready_agents.lock().await.remove(agent_id); + self.last_seen.lock().await.remove(agent_id); + } + } + + /// Applies an agent message only while the sending connection still owns the + /// registration token. The `agents` lock is held across the update so a + /// superseded socket cannot refresh `last_seen`, re-insert readiness, or append + /// playback events after a replacement connection has registered. + async fn apply_agent_message(&self, agent_id: &AgentId, token: u64, event: Envelope) -> bool { + let agents = self.agents.lock().await; + if agents.get(agent_id).is_none_or(|registered| registered.token != token) { + return false; + } + let is_ready = matches!(event.payload, AgentMessage::Ready); + let is_event = matches!(event.payload, AgentMessage::Event { .. }); + if is_ready { + self.ready_agents.lock().await.insert(agent_id.clone()); + } + if is_event { + self.events.lock().await.push(event); + } + self.last_seen.lock().await.insert(agent_id.clone(), Instant::now()); + true + } + + #[must_use] + pub fn run_id(&self) -> RunId { self.run_id.clone() } + + pub async fn dispatch(&self, agent_id: &AgentId, command: Envelope) -> Result<(), TestkitError> { + command.validate()?; + if command.run_id != self.run_id || command.run_generation != self.generation { + return Err(TestkitError::Protocol("command does not belong to the active run generation".to_owned())); + } + let sender = self + .agents + .lock() + .await + .get(agent_id) + .map(|registered| registered.sender.clone()) + .ok_or_else(|| TestkitError::Protocol(format!("agent {} is not connected", agent_id.0)))?; + sender.send(command).await.map_err(|_| TestkitError::Protocol("agent control channel closed".to_owned())) + } + + pub async fn connected_agents(&self) -> Vec { self.agents.lock().await.keys().cloned().collect() } + + pub async fn ready_agents(&self) -> Vec { + const HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(15); + let now = Instant::now(); + let ready = self.ready_agents.lock().await; + let seen = self.last_seen.lock().await; + ready + .iter() + .filter(|agent| { + seen.get(*agent).is_some_and(|last_seen| now.duration_since(*last_seen) <= HEARTBEAT_TIMEOUT) + }) + .cloned() + .collect() + } + + pub async fn take_events(&self) -> Vec> { std::mem::take(&mut *self.events.lock().await) } + + /// Removes only an event belonging to the requested playback. Events for + /// concurrent starts remain queued for their own waiter. + pub async fn take_playback_event(&self, playback_id: &str) -> Option { + let mut events = self.events.lock().await; + let position = events.iter().position(|envelope| { + matches!( + &envelope.payload, + AgentMessage::Event { + event: PlaybackEvent::FirstValidFrame { playback_id: observed, .. } + | PlaybackEvent::HeadersReceived { playback_id: observed, .. } + | PlaybackEvent::Progress { playback_id: observed, .. } + | PlaybackEvent::Terminal { playback_id: observed, .. }, + } if observed.0 == playback_id + ) + })?; + match events.remove(position).payload { + AgentMessage::Event { event } => Some(event), + AgentMessage::Hello { .. } | AgentMessage::Ready | AgentMessage::Heartbeat => None, + } + } +} + +pub async fn controller_websocket(State(state): State, websocket: WebSocketUpgrade) -> Response { + std::future::ready(()).await; + websocket.on_upgrade(move |socket| serve_agent(socket, state)).into_response() +} + +async fn serve_agent(socket: WebSocket, state: ControllerState) { + let (mut sink, mut stream) = socket.split(); + let Some(Ok(Message::Text(first))) = stream.next().await else { return }; + let Ok(hello) = serde_json::from_str::>(&first) else { return }; + if hello.validate().is_err() || hello.run_id != state.run_id || hello.run_generation != state.generation { + return; + } + let AgentMessage::Hello { .. } = hello.payload else { return }; + let agent_id = hello.agent_id; + let connection_token = state.next_connection_token(); + let (tx, mut rx) = mpsc::channel(32); + state.register_agent(agent_id.clone(), connection_token, tx).await; + loop { + tokio::select! { + Some(command) = rx.recv() => { + let Ok(encoded) = serde_json::to_string(&command) else { break }; + if sink.send(Message::Text(encoded.into())).await.is_err() { break; } + } + incoming = stream.next() => match incoming { + Some(Ok(Message::Close(_)) | Err(_)) | None => break, + Some(Ok(Message::Text(text))) => { + if let Ok(event) = serde_json::from_str::>(&text) { + if event.validate().is_ok() + && event.run_id == state.run_id + && event.run_generation == state.generation + && event.agent_id == agent_id + && !state.apply_agent_message(&agent_id, connection_token, event).await + { + // A newer connection replaced this registration; stop + // serving the stale socket before it can mutate state. + break; + } + } + } + _ => {} + }, + } + } + state.unregister_agent(&agent_id, connection_token).await; +} + +pub struct AgentControlConnection { + commands: mpsc::Receiver>, + events: mpsc::Sender>, +} + +impl AgentControlConnection { + pub async fn connect(controller_url: &str, hello: Envelope) -> Result { + hello.validate()?; + if !matches!(hello.payload, AgentMessage::Hello { .. }) { + return Err(TestkitError::Protocol("first agent message must be hello".to_owned())); + } + let (mut socket, _) = + connect_async(controller_url).await.map_err(|error| TestkitError::Protocol(error.to_string()))?; + let encoded = serde_json::to_string(&hello).map_err(|error| TestkitError::Protocol(error.to_string()))?; + socket + .send(ClientMessage::Text(encoded.into())) + .await + .map_err(|error| TestkitError::Protocol(error.to_string()))?; + let (mut sink, mut stream) = socket.split(); + let (command_tx, command_rx) = mpsc::channel(32); + let (event_tx, mut event_rx) = mpsc::channel::>(32); + tokio::spawn(async move { + while let Some(event) = event_rx.recv().await { + let Ok(encoded) = serde_json::to_string(&event) else { break }; + if sink.send(ClientMessage::Text(encoded.into())).await.is_err() { + break; + } + } + }); + tokio::spawn(async move { + while let Some(message) = stream.next().await { + let Ok(message) = message else { break }; + match message { + ClientMessage::Text(text) => { + let Ok(command) = serde_json::from_str::>(&text) else { continue }; + if command.validate().is_err() || command_tx.send(command).await.is_err() { + break; + } + } + ClientMessage::Close(_) => break, + _ => {} + } + } + }); + Ok(Self { commands: command_rx, events: event_tx }) + } + + pub async fn next_command(&mut self) -> Result>, TestkitError> { + Ok(self.commands.recv().await) + } + + pub async fn send_event(&self, event: Envelope) -> Result<(), TestkitError> { + event.validate()?; + self.events.send(event).await.map_err(|_| TestkitError::Protocol("agent event channel closed".to_owned())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{routing::get, Router}; + + #[tokio::test] + async fn unknown_agent_cannot_receive_commands() { + let state = SharedControllerState::new(RunId::new("run"), 1); + let command = Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "command".to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: Command::StopAll, + }; + assert!(state.dispatch(&AgentId::new("missing"), command).await.is_err()); + } + + #[tokio::test] + async fn stale_connection_cleanup_does_not_remove_replacement_registration() { + let state = SharedControllerState::new(RunId::new("run"), 1); + let agent_id = AgentId::new("agent"); + + let (old_tx, _old_rx) = mpsc::channel(1); + let old_token = state.next_connection_token(); + state.register_agent(agent_id.clone(), old_token, old_tx).await; + assert!(state.connected_agents().await.contains(&agent_id)); + + let (new_tx, _new_rx) = mpsc::channel(1); + let new_token = state.next_connection_token(); + state.register_agent(agent_id.clone(), new_token, new_tx).await; + + // The previous connection's cleanup must not delete the replacement. + state.unregister_agent(&agent_id, old_token).await; + assert!(state.connected_agents().await.contains(&agent_id)); + + state.unregister_agent(&agent_id, new_token).await; + assert!(!state.connected_agents().await.contains(&agent_id)); + } + + #[tokio::test] + async fn superseded_connection_cannot_update_state_or_append_events() { + let state = SharedControllerState::new(RunId::new("run"), 1); + let agent_id = AgentId::new("agent"); + + let (old_tx, _old_rx) = mpsc::channel(1); + let old_token = state.next_connection_token(); + state.register_agent(agent_id.clone(), old_token, old_tx).await; + + let (new_tx, _new_rx) = mpsc::channel(1); + let new_token = state.next_connection_token(); + state.register_agent(agent_id.clone(), new_token, new_tx).await; + + let ready = |sequence: u64| Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: format!("ready-{sequence}"), + agent_id: agent_id.clone(), + agent_boot_id: "boot".to_owned(), + source_sequence: sequence, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Ready, + }; + let event = Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "event".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "boot".to_owned(), + source_sequence: 9, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Event { + event: crate::protocol::PlaybackEvent::Terminal { + playback_id: crate::protocol::PlaybackId::new("playback"), + outcome: "passed".to_owned(), + typed_outcome: None, + }, + }, + }; + + // A superseded socket cannot mark the agent ready or append events. + assert!(!state.apply_agent_message(&agent_id, old_token, ready(1)).await); + assert!(!state.apply_agent_message(&agent_id, old_token, event).await); + assert!(matches!(state.ready_agents().await.as_slice(), [])); + assert!(matches!(state.take_events().await.as_slice(), [])); + + // The active connection still can. + assert!(state.apply_agent_message(&agent_id, new_token, ready(2)).await); + assert!(state.ready_agents().await.contains(&agent_id)); + } + + #[tokio::test] + async fn agent_receives_generation_bound_command_over_websocket() { + let state: ControllerState = Arc::new(SharedControllerState::new(RunId::new("run"), 1)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().route("/agent", get(controller_websocket)).with_state(state.clone()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let agent_id = AgentId::new("agent"); + let hello = Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "hello".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Hello { + hostname: "test".to_owned(), + supported_protocols: vec!["ts".to_owned()], + maximum_listeners: 1, + }, + }; + let mut agent = AgentControlConnection::connect(&format!("ws://{address}/agent"), hello).await.unwrap(); + for _ in 0..10 { + if !state.connected_agents().await.is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + agent + .send_event(Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "ready".to_owned(), + agent_id: agent_id.clone(), + agent_boot_id: "boot".to_owned(), + source_sequence: 2, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Ready, + }) + .await + .unwrap(); + for _ in 0..10 { + if state.ready_agents().await.contains(&agent_id) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(state.ready_agents().await.contains(&agent_id)); + let command = Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "stop".to_owned(), + agent_id: AgentId::new("controller"), + agent_boot_id: "controller-boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: Command::StopAll, + }; + state.dispatch(&agent_id, command).await.unwrap(); + assert!(matches!(agent.next_command().await, Ok(Some(Envelope { payload: Command::StopAll, .. })))); + agent + .send_event(Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: "event".to_owned(), + agent_id, + agent_boot_id: "boot".to_owned(), + source_sequence: 3, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Event { + event: crate::protocol::PlaybackEvent::Terminal { + playback_id: crate::protocol::PlaybackId::new("playback"), + outcome: "passed".to_owned(), + typed_outcome: None, + }, + }, + }) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + assert_eq!(state.take_events().await.len(), 1); + server.abort(); + } + + #[tokio::test] + async fn waiting_for_one_playback_keeps_other_playback_events() { + let state = SharedControllerState::new(RunId::new("run"), 1); + for playback_id in ["first", "second"] { + state.events.lock().await.push(Envelope { + schema_version: 1, + run_id: RunId::new("run"), + run_generation: 1, + message_id: format!("event-{playback_id}"), + agent_id: AgentId::new("agent"), + agent_boot_id: "boot".to_owned(), + source_sequence: 1, + caused_by_command_id: None, + local_elapsed_nanos: 0, + payload: AgentMessage::Event { + event: PlaybackEvent::FirstValidFrame { + playback_id: crate::protocol::PlaybackId::new(playback_id), + sequence: 0, + }, + }, + }); + } + assert!(matches!( + state.take_playback_event("second").await, + Some(PlaybackEvent::FirstValidFrame { playback_id, .. }) if playback_id.0 == "second" + )); + assert!(matches!( + state.take_playback_event("first").await, + Some(PlaybackEvent::FirstValidFrame { playback_id, .. }) if playback_id.0 == "first" + )); + } +} diff --git a/tools/tuliprox-testkit/src/vod.rs b/tools/tuliprox-testkit/src/vod.rs new file mode 100644 index 000000000..461cb910a --- /dev/null +++ b/tools/tuliprox-testkit/src/vod.rs @@ -0,0 +1,358 @@ +use crate::TestkitError; +use bytes::Bytes; +use futures::Stream; +use reqwest::{header, StatusCode}; +use std::{pin::Pin, time::Duration}; +use tokio::sync::oneshot; + +pub const DEFAULT_VOD_SIZE: u64 = 128 * 1024 * 1024; // 128 MiB virtual media file +const BLOCK_SIZE: usize = 1024; + +pub async fn read_range(url: &str, start: u64, end_inclusive: u64) -> Result, TestkitError> { + if start > end_inclusive { + return Err(TestkitError::Configuration("VOD range start exceeds end".to_owned())); + } + let range = format!("bytes={start}-{end_inclusive}"); + let response = reqwest::Client::new().get(url).header(header::RANGE, range).send().await?; + if response.status() != StatusCode::PARTIAL_CONTENT { + return Err(TestkitError::Protocol(format!("VOD range returned {}, expected 206", response.status()))); + } + let expected_range = format!("bytes {start}-{end_inclusive}/"); + let content_range = response + .headers() + .get(header::CONTENT_RANGE) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| TestkitError::Protocol("VOD range response has no Content-Range".to_owned()))?; + if !content_range.starts_with(&expected_range) { + return Err(TestkitError::Protocol(format!("unexpected VOD Content-Range {content_range}"))); + } + let body = response.bytes().await?.to_vec(); + let expected_len = usize::try_from(end_inclusive.saturating_sub(start).saturating_add(1)) + .map_err(|_| TestkitError::Configuration("VOD range is too large for this host".to_owned()))?; + if body.len() != expected_len { + return Err(TestkitError::Protocol(format!("VOD range has {} bytes, expected {expected_len}", body.len()))); + } + Ok(body) +} + +#[must_use] +pub fn parse_range_header(value: &str, total_length: u64) -> Option<(u64, u64)> { + let bytes = value.strip_prefix("bytes=")?; + if let Some(suffix) = bytes.strip_prefix('-') { + let suffix_len = suffix.parse::().ok()?; + if suffix_len == 0 || suffix_len > total_length { + return Some((0, total_length)); + } + let start = total_length.saturating_sub(suffix_len); + return Some((start, total_length)); + } + + let (start_str, end_str) = bytes.split_once('-')?; + let start = start_str.parse::().ok()?; + if start >= total_length { + return None; + } + + let end = if end_str.is_empty() { + total_length + } else { + let parsed_end = end_str.parse::().ok()?; + parsed_end.min(total_length.saturating_sub(1)).saturating_add(1) + }; + + if start >= end { + return None; + } + + Some((start, end)) +} + +#[must_use] +pub fn parse_content_range(value: &str) -> Option<(u64, u64, Option)> { + let bytes = value.strip_prefix("bytes ")?; + let (range_part, total_part) = bytes.split_once('/')?; + let (start_str, end_str) = range_part.split_once('-')?; + let start = start_str.parse::().ok()?; + let end = end_str.parse::().ok()?; + let total = if total_part == "*" { None } else { Some(total_part.parse::().ok()?) }; + Some((start, end, total)) +} + +#[must_use] +pub fn generate_deterministic_block(object: &str, block_index: u64) -> [u8; BLOCK_SIZE] { + let seed = blake3::hash(object.as_bytes()); + let block_hash = blake3::keyed_hash(seed.as_bytes(), &block_index.to_le_bytes()); + let mut block = [0_u8; BLOCK_SIZE]; + block[0..8].copy_from_slice(&block_index.to_be_bytes()); + let hash_bytes = block_hash.as_bytes(); + for chunk in block[8..].chunks_mut(32) { + let len = chunk.len().min(32); + chunk.copy_from_slice(&hash_bytes[..len]); + } + block +} + +#[must_use] +pub fn generate_deterministic_slice(object: &str, offset: u64, len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let mut current_offset = offset; + let end_offset = offset.saturating_add(len as u64); + + while current_offset < end_offset { + let block_index = current_offset / (BLOCK_SIZE as u64); + let in_block = usize::try_from(current_offset % (BLOCK_SIZE as u64)).unwrap_or(0); + let block = generate_deterministic_block(object, block_index); + let remaining_in_block = BLOCK_SIZE.saturating_sub(in_block); + let remaining_requested = usize::try_from(end_offset.saturating_sub(current_offset)).unwrap_or(0); + let take = remaining_in_block.min(remaining_requested); + out.extend_from_slice(&block[in_block..in_block.saturating_add(take)]); + current_offset = current_offset.saturating_add(take as u64); + } + out +} + +#[must_use] +pub fn verify_deterministic_slice(object: &str, offset: u64, data: &[u8]) -> bool { + let expected = generate_deterministic_slice(object, offset, data.len()); + data == expected.as_slice() +} + +#[derive(Debug, Clone)] +pub struct VodStreamOptions { + pub object: String, + pub start_offset: u64, + pub length: u64, + pub bitrate: u32, + pub chunk_size: usize, + pub stall_ms: Option, + pub abort_after_bytes: Option, +} + +impl Default for VodStreamOptions { + fn default() -> Self { + Self { + object: "movie.mkv".to_owned(), + start_offset: 0, + length: DEFAULT_VOD_SIZE, + bitrate: 0, + chunk_size: 64 * 1024, + stall_ms: None, + abort_after_bytes: None, + } + } +} + +struct VodStreamState { + options: VodStreamOptions, + evict_rx: oneshot::Receiver<()>, + evicted_flag: Option>, + current_offset: u64, + end_offset: u64, + total_emitted: u64, + interval: Option, + initial_stall_done: bool, +} + +pub fn create_vod_stream( + options: VodStreamOptions, + evict_rx: oneshot::Receiver<()>, +) -> Pin> + Send>> { + create_vod_stream_with_evicted(options, evict_rx, None) +} + +pub fn create_vod_stream_with_evicted( + options: VodStreamOptions, + evict_rx: oneshot::Receiver<()>, + evicted_flag: Option>, +) -> Pin> + Send>> { + let current_offset = options.start_offset; + let end_offset = options.start_offset.saturating_add(options.length); + + let interval = if options.bitrate > 0 { + let bits_per_chunk = (options.chunk_size as u128).saturating_mul(8); + let nanos = + bits_per_chunk.saturating_mul(1_000_000_000).checked_div(u128::from(options.bitrate.max(1))).unwrap_or(1); + let duration = Duration::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX).max(1)); + let mut interval = tokio::time::interval(duration); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + Some(interval) + } else { + None + }; + + let state = VodStreamState { + options, + evict_rx, + evicted_flag, + current_offset, + end_offset, + total_emitted: 0, + interval, + initial_stall_done: false, + }; + + Box::pin(futures::stream::unfold(state, |mut state| async move { + if !state.initial_stall_done { + state.initial_stall_done = true; + if let Some(stall) = state.options.stall_ms { + let stall = tokio::time::sleep(Duration::from_millis(stall)); + tokio::pin!(stall); + tokio::select! { + biased; + res = &mut state.evict_rx => { + if res.is_ok() { + if let Some(flag) = state.evicted_flag.as_ref() { + flag.store(true, std::sync::atomic::Ordering::Release); + } + } + return None; + } + () = &mut stall => {} + } + } + } + + if state.current_offset >= state.end_offset { + return None; + } + + if let Some(abort_limit) = state.options.abort_after_bytes { + if state.total_emitted >= abort_limit { + return None; + } + } + + let remaining = usize::try_from(state.end_offset.saturating_sub(state.current_offset)).unwrap_or(0); + let mut take = state.options.chunk_size.min(remaining); + if let Some(abort_limit) = state.options.abort_after_bytes { + let bytes_until_abort = usize::try_from(abort_limit.saturating_sub(state.total_emitted)).unwrap_or(0); + take = take.min(bytes_until_abort); + } + + if take == 0 { + return None; + } + + if let Some(interval) = state.interval.as_mut() { + tokio::select! { + biased; + res = &mut state.evict_rx => { + if res.is_ok() { + if let Some(flag) = state.evicted_flag.as_ref() { + flag.store(true, std::sync::atomic::Ordering::Release); + } + } + return None; + } + _ = interval.tick() => {} + } + } else { + match state.evict_rx.try_recv() { + Ok(()) => { + if let Some(flag) = state.evicted_flag.as_ref() { + flag.store(true, std::sync::atomic::Ordering::Release); + } + return None; + } + Err(oneshot::error::TryRecvError::Closed) => return None, + Err(oneshot::error::TryRecvError::Empty) => {} + } + } + + let chunk = generate_deterministic_slice(&state.options.object, state.current_offset, take); + state.current_offset = state.current_offset.saturating_add(take as u64); + state.total_emitted = state.total_emitted.saturating_add(take as u64); + + Some((Ok(Bytes::from(chunk)), state)) + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio_stream::StreamExt; + + #[test] + fn parses_open_and_closed_and_suffix_ranges() { + assert_eq!(parse_range_header("bytes=0-1023", 10_000), Some((0, 1024))); + assert_eq!(parse_range_header("bytes=100-", 10_000), Some((100, 10_000))); + assert_eq!(parse_range_header("bytes=-500", 10_000), Some((9500, 10_000))); + assert_eq!(parse_range_header("bytes=10000-", 10_000), None); + assert_eq!(parse_range_header("bytes=500-200", 10_000), None); + } + + #[test] + fn deterministic_slice_verification_is_exact() { + let object = "test_vod.mkv"; + let offset = 4096; + let slice = generate_deterministic_slice(object, offset, 2048); + assert_eq!(slice.len(), 2048); + assert!(verify_deterministic_slice(object, offset, &slice)); + + // Tampered byte must fail verification + let mut tampered = slice.clone(); + tampered[10] ^= 0xFF; + assert!(!verify_deterministic_slice(object, offset, &tampered)); + + // Wrong offset must fail verification + assert!(!verify_deterministic_slice(object, offset + 1, &slice)); + } + + #[tokio::test] + async fn vod_stream_respects_abort_after_bytes() { + let options = VodStreamOptions { + object: "movie.mkv".to_owned(), + start_offset: 0, + length: 100_000, + bitrate: 0, + chunk_size: 1024, + stall_ms: None, + abort_after_bytes: Some(2500), + }; + let (_tx, rx) = oneshot::channel(); + let mut stream = create_vod_stream(options, rx); + + let mut collected = Vec::new(); + while let Some(Ok(chunk)) = stream.next().await { + collected.extend_from_slice(&chunk); + } + + assert_eq!(collected.len(), 2500); + assert!(verify_deterministic_slice("movie.mkv", 0, &collected)); + } + + #[tokio::test] + async fn vod_stream_stops_on_eviction() { + let options = VodStreamOptions { + object: "movie.mkv".to_owned(), + start_offset: 0, + length: 1_000_000, + bitrate: 0, + chunk_size: 1024, + stall_ms: None, + abort_after_bytes: None, + }; + let (tx, rx) = oneshot::channel(); + let mut stream = create_vod_stream(options, rx); + + // Read 1 chunk + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.len(), 1024); + + // Send eviction signal + let _ = tx.send(()); + + // Next read must terminate + assert!(stream.next().await.is_none()); + } + + #[test] + fn parse_content_range_valid_and_invalid() { + assert_eq!(parse_content_range("bytes 0-65535/134217728"), Some((0, 65535, Some(134_217_728)))); + assert_eq!(parse_content_range("bytes 100-200/*"), Some((100, 200, None))); + assert_eq!(parse_content_range("bytes 0-0/1"), Some((0, 0, Some(1)))); + assert_eq!(parse_content_range("0-65535/134217728"), None); + assert_eq!(parse_content_range("bytes 0-65535"), None); + assert_eq!(parse_content_range("bytes invalid-range/100"), None); + } +}