mirror of
https://github.com/euzu/tuliprox.git
synced 2026-10-27 17:22:39 +01:00
* fix(hls): keep provider affinity after reconnect lease expiry - Remember the provider that last delivered media for reconnect-capable playback (HLS, DASH, Catchup, VOD) beyond the lease window. The preference holds no slot, never forces grace and falls back to the lineup when the provider is full. - End it on provider failure, preemption, kick, timeout or admin terminate; stale cleanups are rejected by binding tag, request ID and session token. Retry token claims stay bounded. - Terminate and kick now release the HLS binding they acquired. - Add reverse_proxy.stream.provider_affinity_ttl_secs (default 120, max 86400) with Web UI field, docs and translations. - Testkit: pause_millis steps, TTL overrides and the m3u-hls-provider-affinity-after-lease-expiry scenario. * fix(hls): keep provider affinity after reconnect lease expiry - Remember the provider that last delivered media for reconnect-capable playback (HLS, DASH, Catchup, VOD) beyond the lease window. The preference holds no slot, never forces grace and falls back to the lineup when the provider is full. - End it on provider failure, preemption, kick, timeout or admin terminate; stale cleanups are rejected by binding tag, request ID and session token. Retry token claims stay bounded. - Terminate and kick now release the HLS binding they acquired. - Add reverse_proxy.stream.provider_affinity_ttl_secs (default 120, max 86400) with Web UI field, docs and translations. - Testkit: pause_millis steps, TTL overrides and the m3u-hls-provider-affinity-after-lease-expiry scenario. * fix(hls): keep provider affinity after reconnect lease expiry - Remember the provider that last delivered media for reconnect-capable playback (HLS, DASH, Catchup, VOD) beyond the lease window. The preference holds no slot, never forces grace and falls back to the lineup when the provider is full. - End it on provider failure, preemption, kick, timeout or admin terminate; stale cleanups are rejected by binding tag, request ID and session token. Retry token claims stay bounded. - Terminate and kick now release the HLS binding they acquired. - Add reverse_proxy.stream.provider_affinity_ttl_secs (default 120, max 86400) with Web UI field, docs and translations. - Testkit: pause_millis steps, TTL overrides and the m3u-hls-provider-affinity-after-lease-expiry scenario. --------- Co-authored-by: euzu <euzu@proton.me>
6169 lines
270 KiB
Rust
6169 lines
270 KiB
Rust
use crate::{
|
|
provider_leases::{ProviderAffinityTtl, ProviderLeaseTable, ProviderLeaseUsage},
|
|
provider_lineup_manager::ProviderLineupManager,
|
|
EventManager, SharedStreamManager,
|
|
};
|
|
use log::error;
|
|
use shared::utils::sanitize_sensitive_info;
|
|
use std::{
|
|
cmp::Reverse,
|
|
collections::{BTreeMap, HashMap, HashSet},
|
|
net::SocketAddr,
|
|
sync::{
|
|
atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering},
|
|
Arc, LazyLock, OnceLock, RwLockReadGuard, RwLockWriteGuard, Weak,
|
|
},
|
|
time::{Duration, Instant},
|
|
};
|
|
use tokio::time::Instant as TokioInstant;
|
|
use tokio_util::sync::CancellationToken;
|
|
use tuliprox_core::{
|
|
model::{
|
|
AllocationId, AppConfig, ConfigInput, ConnectionLifecycle, GracePeriodOptions, PlaybackKind, PlaybackLeaseId,
|
|
PlaybackRequestId, PlaybackRequestOutcome, PlaybackSelectionReason, ProviderAllocation, ProviderBindingTag,
|
|
ProviderCloseReason, ProviderConfig, ProviderHandle, SharedSubscriberId,
|
|
},
|
|
utils::debug_if_enabled,
|
|
};
|
|
|
|
static DUMMY_ADDR: LazyLock<SocketAddr> = LazyLock::new(|| SocketAddr::from(([127, 0, 0, 1], 0)));
|
|
type SharedConnectionId = AllocationId;
|
|
type PreemptionCandidate = (PriorityOwner, AllocationId, i8, Instant);
|
|
// Key for BTreeMap priority index: (priority, Reverse<created_at>, AllocationId)
|
|
// Semantics: lower numeric priority value = higher importance (0 = highest, 127 = lowest).
|
|
// `.last()` on the BTreeMap returns the entry with the highest priority value, which is
|
|
// the lowest-importance connection and therefore the best eviction victim.
|
|
// Ties are broken by `Reverse<Instant>`: among equal priority values, the oldest connection
|
|
// (smallest `created_at`) sorts last and is evicted first.
|
|
type PriorityKey = (i8, Reverse<Instant>, AllocationId);
|
|
|
|
const PREEMPTION_COMPLETION_TIMEOUT: Duration = Duration::from_millis(1500);
|
|
const EVICTED_PROVIDER_RELEASE_POLL_INTERVAL: Duration = Duration::from_millis(10);
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub(crate) struct ProviderReleaseSnapshot {
|
|
pub addr: SocketAddr,
|
|
single_allocations: Vec<AllocationId>,
|
|
shared_subscribers: Vec<SharedSubscriberId>,
|
|
}
|
|
|
|
impl ProviderReleaseSnapshot {
|
|
pub fn is_empty(&self) -> bool { self.single_allocations.is_empty() && self.shared_subscribers.is_empty() }
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum ConnectionKind {
|
|
Normal,
|
|
Soft,
|
|
}
|
|
|
|
fn is_better_preemption_candidate(current: Option<PreemptionCandidate>, candidate: PreemptionCandidate) -> bool {
|
|
match current {
|
|
None => true,
|
|
Some((_, current_allocation_id, current_priority, current_created_at)) => {
|
|
candidate.2 > current_priority
|
|
|| (candidate.2 == current_priority
|
|
&& (candidate.3 < current_created_at
|
|
|| (candidate.3 == current_created_at && candidate.1 < current_allocation_id)))
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum PriorityOwner {
|
|
Single(AllocationId),
|
|
Shared(SharedConnectionId),
|
|
}
|
|
|
|
/// Resolves a public live HLS session token to its stable provider playback owner.
|
|
///
|
|
/// The random suffix distinguishes public sessions, while the preceding client,
|
|
/// user and channel fingerprint identifies one provider playback across retries.
|
|
/// Catchup and shared-cache owners retain their own identity semantics.
|
|
fn playback_lease_owner(owner: &str) -> &str {
|
|
if owner.starts_with("m3u-catchup|") || owner.starts_with("catchup|") || owner.starts_with("hls-cache:") {
|
|
return owner;
|
|
}
|
|
let Some((base, suffix)) = owner.rsplit_once("|hls|") else {
|
|
return owner;
|
|
};
|
|
if suffix.len() == 16 && suffix.bytes().all(|byte| byte.is_ascii_alphanumeric()) {
|
|
base
|
|
} else {
|
|
owner
|
|
}
|
|
}
|
|
|
|
/// How an acquisition of a playback owner is tied to a provider.
|
|
enum OwnerProviderPin {
|
|
/// A confirmed or active lease pins the playback to exactly this provider.
|
|
Reserved(Arc<str>),
|
|
/// A recently confirmed playback prefers this provider; it holds no capacity.
|
|
Preferred(Arc<str>),
|
|
}
|
|
|
|
/// Playback identity of one provider acquisition.
|
|
///
|
|
/// `owner` carries the public session identity so transport cleanup can distinguish
|
|
/// retries. Provider leases use [`Self::provider_owner`] to resolve it to a stable
|
|
/// client/user/channel owner and therefore one capacity slot.
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub struct PlaybackLeaseRef<'a> {
|
|
pub owner: &'a str,
|
|
pub kind: PlaybackKind,
|
|
pub request_id: PlaybackRequestId,
|
|
}
|
|
|
|
impl<'a> PlaybackLeaseRef<'a> {
|
|
pub fn new(owner: &'a str, kind: PlaybackKind) -> Self {
|
|
Self { owner, kind, request_id: PlaybackRequestId::next() }
|
|
}
|
|
|
|
/// Stable provider lease identity, independent of the public HLS token suffix.
|
|
pub fn provider_owner(&self) -> &'a str { playback_lease_owner(self.owner) }
|
|
|
|
/// Compatibility view of a bare session owner: one request, live TS semantics.
|
|
pub fn for_session_owner(owner: &'a str) -> Self { Self::new(owner, PlaybackKind::LiveTs) }
|
|
}
|
|
|
|
struct AcquireProviderParams<'a> {
|
|
addr: &'a SocketAddr,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
lease: Option<PlaybackLeaseRef<'a>>,
|
|
}
|
|
|
|
struct ProviderAllocationGuard(Option<ProviderAllocation>);
|
|
|
|
impl ProviderAllocationGuard {
|
|
fn new(allocation: ProviderAllocation) -> Self { Self(Some(allocation)) }
|
|
|
|
fn allocation(&self) -> &ProviderAllocation { self.0.as_ref().unwrap_or(&ProviderAllocation::Exhausted) }
|
|
|
|
fn take(&mut self) -> ProviderAllocation { self.0.take().unwrap_or(ProviderAllocation::Exhausted) }
|
|
}
|
|
|
|
impl Drop for ProviderAllocationGuard {
|
|
fn drop(&mut self) {
|
|
if let Some(allocation) = self.0.take() {
|
|
allocation.release();
|
|
}
|
|
}
|
|
}
|
|
|
|
impl AcquireProviderParams<'_> {
|
|
#[inline]
|
|
fn session_owner(&self) -> Option<&str> { self.lease.map(|lease| lease.provider_owner()) }
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct SharedAllocation {
|
|
allocation_id: AllocationId,
|
|
origin_subscriber_id: SharedSubscriberId,
|
|
allocation: ProviderAllocation,
|
|
/// Keyed by unique subscriber id, never by socket: two external clients behind one
|
|
/// reverse proxy must not collapse into a single entry.
|
|
connections: HashMap<SharedSubscriberId, SharedSubscriber>,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
created_at: Instant,
|
|
cancel_token: Option<CancellationToken>,
|
|
session_owner: Option<Arc<str>>,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct ActiveConnectionInfo {
|
|
allocation_id: AllocationId,
|
|
client_addr: SocketAddr,
|
|
allocation: ProviderAllocation,
|
|
// Used to signal preemption to the consumer of this connection
|
|
cancel_token: CancellationToken,
|
|
completion_token: CancellationToken,
|
|
close_reason: Arc<AtomicU8>,
|
|
lifecycle: ConnectionLifecycle,
|
|
has_body_owner: bool,
|
|
reaper_spawned: bool,
|
|
open_generation: u64,
|
|
created_at: Instant,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
session_owner: Option<Arc<str>>,
|
|
// Public session identity scopes targeted socket cleanup independently of the
|
|
// stable provider lease owner shared by entry retries.
|
|
request_owner: Option<Arc<str>>,
|
|
playback_request_id: Option<PlaybackRequestId>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
struct SharedSubscriber {
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
/// Transport metadata and socket-wide close target only; never an identity key.
|
|
addr: SocketAddr,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default)]
|
|
struct SharedConnections {
|
|
by_key: HashMap<Arc<str>, SharedAllocation>,
|
|
key_by_subscriber: HashMap<SharedSubscriberId, Arc<str>>,
|
|
shared_by_allocation_id: HashMap<AllocationId, Arc<str>>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default)]
|
|
struct Connections {
|
|
// Primary map keyed by AllocationId, never by socket address.
|
|
single: HashMap<AllocationId, ActiveConnectionInfo>,
|
|
// Secondary index for socket-wide transport close/kick actions.
|
|
single_by_addr: HashMap<SocketAddr, HashSet<AllocationId>>,
|
|
shared: SharedConnections,
|
|
// Index to quickly find connections by provider name for preemption
|
|
// ProviderName -> Set<AllocationId>
|
|
by_provider: HashMap<Arc<str>, HashSet<AllocationId>>,
|
|
// Index to find every allocation (single and shared) owned by a playback session.
|
|
// SessionOwner -> Set<AllocationId>
|
|
by_owner: HashMap<Arc<str>, HashSet<AllocationId>>,
|
|
// Priority index per provider alias for O(log n) victim lookup
|
|
// ProviderName -> BTreeMap<PriorityKey, PriorityOwner>
|
|
priority_index: HashMap<Arc<str>, BTreeMap<PriorityKey, PriorityOwner>>,
|
|
soft_priority_index: HashMap<Arc<str>, BTreeMap<PriorityKey, PriorityOwner>>,
|
|
}
|
|
|
|
pub struct ManagedProviderHandle {
|
|
manager: Arc<ActiveProviderManager>,
|
|
handle: Option<ProviderHandle>,
|
|
}
|
|
|
|
impl std::fmt::Debug for ManagedProviderHandle {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("ManagedProviderHandle").field("handle", &self.handle).finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl ManagedProviderHandle {
|
|
#[inline]
|
|
pub fn manager(&self) -> &Arc<ActiveProviderManager> { &self.manager }
|
|
|
|
#[inline]
|
|
pub fn new(manager: Arc<ActiveProviderManager>, handle: ProviderHandle) -> Self {
|
|
Self { manager, handle: Some(handle) }
|
|
}
|
|
|
|
#[inline]
|
|
pub fn handle(&self) -> Option<&ProviderHandle> { self.handle.as_ref() }
|
|
|
|
#[inline]
|
|
pub fn take(&mut self) -> Option<ProviderHandle> { self.handle.take() }
|
|
|
|
#[inline]
|
|
pub fn disarm(&mut self) -> Option<ProviderHandle> { self.handle.take() }
|
|
|
|
pub fn mark_opening(&self) {
|
|
if let Some(handle) = self.handle.as_ref() {
|
|
self.manager.mark_opening(handle.allocation_id);
|
|
}
|
|
}
|
|
|
|
pub fn register_body_owner(&self) -> bool {
|
|
if let Some(handle) = self.handle.as_ref() {
|
|
self.manager.register_body_owner(handle.allocation_id)
|
|
} else {
|
|
false
|
|
}
|
|
}
|
|
|
|
pub fn renew_opening_tokens(&mut self) -> Option<(CancellationToken, CancellationToken)> {
|
|
if let Some(ref mut handle) = self.handle {
|
|
if let Some((cancel, completion, gen)) = self.manager.renew_opening_tokens(handle.allocation_id) {
|
|
handle.cancel_token = Some(cancel.clone());
|
|
handle.completion_token = Some(completion.clone());
|
|
handle.open_generation = gen;
|
|
return Some((cancel, completion));
|
|
}
|
|
}
|
|
None
|
|
}
|
|
}
|
|
|
|
impl Drop for ManagedProviderHandle {
|
|
fn drop(&mut self) {
|
|
if let Some(handle) = self.handle.take() {
|
|
self.manager.release_handle(&handle);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub struct ActiveProviderManagerCore {
|
|
// Serializes capacity transitions across counters, allocation indices and leases.
|
|
capacity_transition: std::sync::Mutex<()>,
|
|
is_shutting_down: AtomicBool,
|
|
shutdown_token: CancellationToken,
|
|
pub(crate) providers: ProviderLineupManager,
|
|
connections: std::sync::RwLock<Connections>,
|
|
leases: std::sync::RwLock<ProviderLeaseTable>,
|
|
next_allocation_id: AtomicU64,
|
|
}
|
|
|
|
impl ActiveProviderManagerCore {
|
|
/// Serialises capacity transitions (acquire/release/reclassify) so a reclassify
|
|
/// cannot race an acquire or release.
|
|
///
|
|
/// Lock-order invariant: this lock must be acquired **before** the connections
|
|
/// lock (`read_connections`/`write_connections`). No path may hold the
|
|
/// connections lock and then call this; doing so can deadlock.
|
|
fn lock_capacity_transition(&self) -> std::sync::MutexGuard<'_, ()> {
|
|
match self.capacity_transition.lock() {
|
|
Ok(guard) => guard,
|
|
Err(poisoned) => {
|
|
error!("Recovering poisoned active-provider capacity transition lock");
|
|
poisoned.into_inner()
|
|
}
|
|
}
|
|
}
|
|
|
|
fn read_connections(&self) -> RwLockReadGuard<'_, Connections> {
|
|
match self.connections.read() {
|
|
Ok(guard) => guard,
|
|
Err(poisoned) => {
|
|
error!("Recovering poisoned active-provider connection lock");
|
|
poisoned.into_inner()
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write_connections(&self) -> RwLockWriteGuard<'_, Connections> {
|
|
match self.connections.write() {
|
|
Ok(guard) => guard,
|
|
Err(poisoned) => {
|
|
error!("Recovering poisoned active-provider connection lock");
|
|
poisoned.into_inner()
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write_leases(&self) -> RwLockWriteGuard<'_, ProviderLeaseTable> {
|
|
match self.leases.write() {
|
|
Ok(guard) => guard,
|
|
Err(poisoned) => {
|
|
error!("Recovering poisoned provider-lease lock");
|
|
poisoned.into_inner()
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn read_leases(&self) -> RwLockReadGuard<'_, ProviderLeaseTable> {
|
|
match self.leases.read() {
|
|
Ok(guard) => guard,
|
|
Err(poisoned) => poisoned.into_inner(),
|
|
}
|
|
}
|
|
|
|
pub fn complete_release(&self, alloc_id: AllocationId) { self.complete_release_with_generation(alloc_id, None); }
|
|
|
|
pub fn complete_release_with_generation(&self, alloc_id: AllocationId, generation: Option<u64>) {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
Self::complete_release_locked(&mut connections, alloc_id, generation);
|
|
}
|
|
|
|
fn complete_release_locked(
|
|
connections: &mut Connections,
|
|
alloc_id: AllocationId,
|
|
expected_generation: Option<u64>,
|
|
) -> Option<ProviderAllocation> {
|
|
if let Some(info) = connections.single.get(&alloc_id) {
|
|
if let Some(expected_gen) = expected_generation {
|
|
if info.open_generation != expected_gen {
|
|
return None;
|
|
}
|
|
}
|
|
}
|
|
if let Some(mut info) = connections.single.remove(&alloc_id) {
|
|
ActiveProviderManager::unindex_owner(connections, alloc_id, info.session_owner.as_ref());
|
|
if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) {
|
|
set.remove(&alloc_id);
|
|
if set.is_empty() {
|
|
connections.single_by_addr.remove(&info.client_addr);
|
|
}
|
|
}
|
|
if !info.allocation.is_unlimited_provider() {
|
|
if let Some(name) = info.allocation.get_provider_name() {
|
|
if let Some(list) = connections.by_provider.get_mut(&name) {
|
|
list.remove(&alloc_id);
|
|
}
|
|
ActiveProviderManager::remove_priority_entry(
|
|
connections,
|
|
&name,
|
|
&(info.priority, Reverse(info.created_at), alloc_id),
|
|
info.kind,
|
|
);
|
|
}
|
|
}
|
|
info.lifecycle = ConnectionLifecycle::Closed;
|
|
info.cancel_token.cancel();
|
|
info.allocation.release();
|
|
Some(info.allocation)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
fn plan_single_release_locked(connections: &mut Connections, alloc_id: AllocationId) -> ReleaseAction {
|
|
let Some(info) = connections.single.get_mut(&alloc_id) else {
|
|
return ReleaseAction::None;
|
|
};
|
|
if info.lifecycle == ConnectionLifecycle::Closed {
|
|
return ReleaseAction::None;
|
|
}
|
|
|
|
info.cancel_token.cancel();
|
|
|
|
let completed = info.completion_token.is_cancelled();
|
|
let needs_completion_wait = (info.lifecycle == ConnectionLifecycle::Closing
|
|
|| info.lifecycle == ConnectionLifecycle::Opening
|
|
|| info.has_body_owner)
|
|
&& !completed;
|
|
|
|
if needs_completion_wait {
|
|
info.lifecycle = ConnectionLifecycle::Closing;
|
|
let client_addr = info.client_addr;
|
|
let prio = info
|
|
.allocation
|
|
.get_provider_name()
|
|
.map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind));
|
|
let already_spawned = info.reaper_spawned;
|
|
info.reaper_spawned = true;
|
|
let gen = info.open_generation;
|
|
let completion_token = info.completion_token.clone();
|
|
|
|
if let Some((name, key, kind)) = prio {
|
|
ActiveProviderManager::remove_priority_entry(connections, &name, &key, kind);
|
|
}
|
|
if let Some(set) = connections.single_by_addr.get_mut(&client_addr) {
|
|
set.remove(&alloc_id);
|
|
if set.is_empty() {
|
|
connections.single_by_addr.remove(&client_addr);
|
|
}
|
|
}
|
|
if already_spawned {
|
|
ReleaseAction::None
|
|
} else {
|
|
ReleaseAction::Wait(alloc_id, gen, completion_token)
|
|
}
|
|
} else {
|
|
let gen = info.open_generation;
|
|
Self::complete_release_locked(connections, alloc_id, Some(gen));
|
|
ReleaseAction::Immediate
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
enum ReleaseAction {
|
|
None,
|
|
Immediate,
|
|
Wait(AllocationId, u64, CancellationToken),
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
pub(crate) enum PreemptionOutcome {
|
|
Acquired(ProviderAllocation),
|
|
/// Victim body is still draining. The optional pair is `(alloc_id, open_generation)` of the
|
|
/// victim for an idempotent self-release attempt once the token fires; `None` for shared-stream
|
|
/// teardowns where release happens inside the spawned task and no `complete_release_locked` call
|
|
/// is needed.
|
|
PendingCompletion(Option<(AllocationId, u64)>, CancellationToken),
|
|
Exhausted,
|
|
}
|
|
|
|
pub struct ActiveProviderManager {
|
|
core: Arc<ActiveProviderManagerCore>,
|
|
shared_stream_manager: OnceLock<Weak<SharedStreamManager>>,
|
|
}
|
|
|
|
impl std::ops::Deref for ActiveProviderManager {
|
|
type Target = ActiveProviderManagerCore;
|
|
|
|
#[inline]
|
|
fn deref(&self) -> &Self::Target { &self.core }
|
|
}
|
|
|
|
impl ActiveProviderManager {
|
|
fn has_connections_for_addr(&self, addr: &SocketAddr) -> bool {
|
|
let _transition = self.lock_capacity_transition();
|
|
let connections = self.read_connections();
|
|
connections.single.values().any(|info| info.client_addr == *addr)
|
|
|| connections
|
|
.shared
|
|
.by_key
|
|
.values()
|
|
.any(|shared| shared.connections.values().any(|subscriber| subscriber.addr == *addr))
|
|
}
|
|
|
|
/// Waits for every allocation currently using an address to leave the registry.
|
|
/// Admission handoffs use `wait_for_snapshot_release` to ignore later allocations at that address.
|
|
pub async fn wait_for_addr_release(&self, addr: &SocketAddr, timeout: Duration) -> bool {
|
|
let deadline = TokioInstant::now() + timeout;
|
|
loop {
|
|
if !self.has_connections_for_addr(addr) {
|
|
return true;
|
|
}
|
|
let now = TokioInstant::now();
|
|
if now >= deadline {
|
|
return false;
|
|
}
|
|
tokio::time::sleep_until((now + EVICTED_PROVIDER_RELEASE_POLL_INTERVAL).min(deadline)).await;
|
|
}
|
|
}
|
|
|
|
pub(crate) fn release_snapshot_for_addr(&self, addr: &SocketAddr) -> ProviderReleaseSnapshot {
|
|
let _transition = self.lock_capacity_transition();
|
|
let connections = self.read_connections();
|
|
let single_allocations = connections
|
|
.single
|
|
.values()
|
|
.filter_map(|info| (info.client_addr == *addr).then_some(info.allocation_id))
|
|
.collect();
|
|
let shared_subscribers = connections
|
|
.shared
|
|
.by_key
|
|
.values()
|
|
.flat_map(|shared| {
|
|
shared.connections.iter().filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id))
|
|
})
|
|
.collect();
|
|
ProviderReleaseSnapshot { addr: *addr, single_allocations, shared_subscribers }
|
|
}
|
|
|
|
fn has_connections_from_snapshot(&self, snapshot: &ProviderReleaseSnapshot) -> bool {
|
|
let _transition = self.lock_capacity_transition();
|
|
let connections = self.read_connections();
|
|
snapshot.single_allocations.iter().any(|id| {
|
|
connections.single.contains_key(id)
|
|
|| connections.shared.shared_by_allocation_id.get(id).is_some_and(|key| {
|
|
connections.shared.by_key.get(key).is_some_and(|shared| {
|
|
connections.shared.key_by_subscriber.get(&shared.origin_subscriber_id) == Some(key)
|
|
})
|
|
})
|
|
}) || snapshot.shared_subscribers.iter().any(|id| connections.shared.key_by_subscriber.contains_key(id))
|
|
}
|
|
|
|
/// Waits for the kicked transport's original provider allocations to leave the registry.
|
|
/// The socket close may be signalled before its response bodies and provider handles are dropped.
|
|
pub(crate) async fn wait_for_snapshot_release(
|
|
&self,
|
|
snapshot: &ProviderReleaseSnapshot,
|
|
timeout: Duration,
|
|
) -> bool {
|
|
let deadline = TokioInstant::now() + timeout;
|
|
loop {
|
|
if !self.has_connections_from_snapshot(snapshot) {
|
|
return true;
|
|
}
|
|
let now = TokioInstant::now();
|
|
if now >= deadline {
|
|
return false;
|
|
}
|
|
tokio::time::sleep_until((now + EVICTED_PROVIDER_RELEASE_POLL_INTERVAL).min(deadline)).await;
|
|
}
|
|
}
|
|
|
|
fn upsert_priority_entry(
|
|
connections: &mut Connections,
|
|
provider_name: &Arc<str>,
|
|
key: PriorityKey,
|
|
owner: PriorityOwner,
|
|
kind: ConnectionKind,
|
|
) {
|
|
connections.priority_index.entry(provider_name.clone()).or_default().insert(key, owner);
|
|
if kind == ConnectionKind::Soft {
|
|
connections.soft_priority_index.entry(provider_name.clone()).or_default().insert(key, owner);
|
|
}
|
|
}
|
|
|
|
fn remove_priority_entry(
|
|
connections: &mut Connections,
|
|
provider_name: &Arc<str>,
|
|
key: &PriorityKey,
|
|
kind: ConnectionKind,
|
|
) {
|
|
if let Some(tree) = connections.priority_index.get_mut(provider_name) {
|
|
tree.remove(key);
|
|
}
|
|
if kind == ConnectionKind::Soft {
|
|
if let Some(tree) = connections.soft_priority_index.get_mut(provider_name) {
|
|
tree.remove(key);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn index_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc<str>>) {
|
|
if let Some(owner) = owner {
|
|
connections.by_owner.entry(Arc::clone(owner)).or_default().insert(allocation_id);
|
|
}
|
|
}
|
|
|
|
fn unindex_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc<str>>) {
|
|
if let Some(owner) = owner {
|
|
let remove = connections.by_owner.get_mut(owner).is_some_and(|set| {
|
|
set.remove(&allocation_id);
|
|
set.is_empty()
|
|
});
|
|
if remove {
|
|
connections.by_owner.remove(owner);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn shared_effective_priority(
|
|
subscribers: &HashMap<SharedSubscriberId, SharedSubscriber>,
|
|
kind: ConnectionKind,
|
|
) -> Option<i8> {
|
|
subscribers.values().filter(|subscriber| subscriber.kind == kind).map(|subscriber| subscriber.priority).min()
|
|
}
|
|
|
|
fn shared_effective_kind(subscribers: &HashMap<SharedSubscriberId, SharedSubscriber>) -> ConnectionKind {
|
|
if subscribers.values().all(|subscriber| subscriber.kind == ConnectionKind::Soft) {
|
|
ConnectionKind::Soft
|
|
} else {
|
|
ConnectionKind::Normal
|
|
}
|
|
}
|
|
|
|
pub fn new(cfg: &AppConfig, event_manager: &Arc<EventManager>) -> Self {
|
|
let grace_period_options = Self::get_grace_options(cfg);
|
|
let inputs = Self::get_config_inputs(cfg);
|
|
Self {
|
|
core: Arc::new(ActiveProviderManagerCore {
|
|
capacity_transition: std::sync::Mutex::new(()),
|
|
is_shutting_down: AtomicBool::new(false),
|
|
shutdown_token: CancellationToken::new(),
|
|
providers: ProviderLineupManager::new(inputs, grace_period_options, event_manager),
|
|
connections: std::sync::RwLock::new(Connections::default()),
|
|
leases: std::sync::RwLock::new(ProviderLeaseTable::with_affinity_ttl(Self::get_affinity_ttl(cfg))),
|
|
next_allocation_id: AtomicU64::new(1),
|
|
}),
|
|
shared_stream_manager: OnceLock::new(),
|
|
}
|
|
}
|
|
|
|
/// Routes account observations reported through `event_manager` to this manager.
|
|
/// Only the first binding wins, so test or auxiliary managers cannot hijack the runtime one.
|
|
pub fn bind_event_manager(&self, event_manager: &EventManager) {
|
|
let _ = event_manager.account_provider.set(Arc::downgrade(&self.core));
|
|
}
|
|
|
|
pub fn observe_account(
|
|
&self,
|
|
observation: tuliprox_core::model::ProviderAccountObservation,
|
|
) -> Option<tuliprox_core::model::ProviderAccountObservation> {
|
|
self.core.providers.observe_account(observation)
|
|
}
|
|
|
|
pub fn reset_account_health(&self, name: &Arc<str>) { self.core.providers.forget_account_health(name); }
|
|
pub fn pending_account_observations(&self) -> Vec<tuliprox_core::model::ProviderAccountObservation> {
|
|
self.core.providers.pending_account_observations()
|
|
}
|
|
pub fn is_observation_pending(&self, observation: &tuliprox_core::model::ProviderAccountObservation) -> bool {
|
|
self.core.providers.is_observation_pending(observation)
|
|
}
|
|
pub fn acknowledge_account_observation(&self, observation: &tuliprox_core::model::ProviderAccountObservation) {
|
|
self.core.providers.acknowledge_account_observation(observation);
|
|
}
|
|
pub fn request_account_probe(&self, name: &Arc<str>) { self.core.providers.request_account_probe(name); }
|
|
pub fn is_account_probe_requested(&self, name: &Arc<str>) -> bool {
|
|
self.core.providers.is_account_probe_requested(name)
|
|
}
|
|
pub fn clear_account_probe(&self, name: &Arc<str>) { self.core.providers.clear_account_probe(name); }
|
|
pub async fn account_health_changed(&self) { self.core.providers.account_health_changed.notified().await; }
|
|
|
|
pub fn set_shared_stream_manager(&self, manager: &Arc<SharedStreamManager>) {
|
|
let _ = self.shared_stream_manager.set(Arc::downgrade(manager));
|
|
}
|
|
|
|
/// Closes provider admission and releases all physical allocations and leases.
|
|
pub fn shutdown(&self) {
|
|
self.shutdown_token.cancel();
|
|
self.is_shutting_down.store(true, Ordering::Release);
|
|
let _transition = self.lock_capacity_transition();
|
|
let connections = std::mem::take(&mut *self.write_connections());
|
|
self.write_leases().clear();
|
|
|
|
for info in connections.single.into_values() {
|
|
info.cancel_token.cancel();
|
|
info.allocation.release();
|
|
}
|
|
for shared in connections.shared.by_key.into_values() {
|
|
if let Some(cancel_token) = shared.cancel_token {
|
|
cancel_token.cancel();
|
|
}
|
|
shared.allocation.release();
|
|
}
|
|
self.providers.reconcile_connections(HashMap::new());
|
|
}
|
|
|
|
fn get_config_inputs(cfg: &AppConfig) -> Vec<Arc<ConfigInput>> {
|
|
cfg.sources.load().inputs.iter().filter(|i| i.enabled).map(Arc::clone).collect()
|
|
}
|
|
|
|
fn get_grace_options(cfg: &AppConfig) -> GracePeriodOptions { cfg.config.load().get_grace_options() }
|
|
|
|
fn get_affinity_ttl(cfg: &AppConfig) -> ProviderAffinityTtl {
|
|
ProviderAffinityTtl(cfg.config.load().get_provider_affinity_ttl_secs())
|
|
}
|
|
|
|
pub fn update_config(&self, cfg: &AppConfig) {
|
|
let grace_period_options = Self::get_grace_options(cfg);
|
|
let inputs = Self::get_config_inputs(cfg);
|
|
self.providers.update_config(inputs, &grace_period_options);
|
|
self.write_leases().set_affinity_ttl(Self::get_affinity_ttl(cfg));
|
|
self.reconcile_connections();
|
|
}
|
|
|
|
pub fn reconcile_connections(&self) {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut counts = HashMap::<Arc<str>, usize>::new();
|
|
{
|
|
let connections = self.read_connections();
|
|
|
|
// Single connections
|
|
for info in connections.single.values() {
|
|
if let Some(name) = info.allocation.get_provider_name() {
|
|
*counts.entry(name).or_insert(0) += 1;
|
|
}
|
|
}
|
|
|
|
// Shared connections
|
|
for shared in connections.shared.by_key.values() {
|
|
if let Some(name) = shared.allocation.get_provider_name() {
|
|
*counts.entry(name).or_insert(0) += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
self.providers.reconcile_connections(counts);
|
|
}
|
|
|
|
fn prune_expired_leases(leases: &mut ProviderLeaseTable) { leases.prune(TokioInstant::now()); }
|
|
|
|
fn has_foreign_reservation(&self, provider_name: &Arc<str>, session_owner: Option<&str>) -> bool {
|
|
if !self.providers.reservation_blocks_other_sessions(provider_name) {
|
|
return false;
|
|
}
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
leases.has_foreign_reserved_lease(provider_name, session_owner)
|
|
}
|
|
|
|
/// Resolves how a playback owner is bound to a provider of `input_name`, with a
|
|
/// single lease-table lookup per acquisition.
|
|
fn owner_provider_pin(&self, input_name: &Arc<str>, session_owner: &str) -> Option<OwnerProviderPin> {
|
|
let (lease_pin, affinity_provider) = {
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
let lease_pin = leases
|
|
.lease_of_owner(session_owner)
|
|
.map(|lease| (Arc::clone(&lease.provider_name), lease.state.is_confirmed()));
|
|
(lease_pin, leases.affinity_provider_for_owner(session_owner))
|
|
};
|
|
// The lease lock is released before the connection lock is taken.
|
|
if let Some((provider_name, confirmed)) = lease_pin {
|
|
if self.providers.is_provider_for_input(&provider_name, input_name)
|
|
&& (confirmed || self.has_active_owner_for_provider(&provider_name, session_owner))
|
|
{
|
|
return Some(OwnerProviderPin::Reserved(provider_name));
|
|
}
|
|
}
|
|
affinity_provider
|
|
.filter(|provider_name| self.providers.is_provider_for_input(provider_name, input_name))
|
|
.map(OwnerProviderPin::Preferred)
|
|
}
|
|
|
|
/// Provider a playback currently prefers on re-entry, for diagnostics and tests.
|
|
pub fn provider_affinity_for_owner(&self, session_owner: &str) -> Option<Arc<str>> {
|
|
let session_owner = playback_lease_owner(session_owner);
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
leases.affinity_provider_for_owner(session_owner)
|
|
}
|
|
|
|
/// A provisional lease only pins its provider while its allocation is active.
|
|
/// Once an unstarted request releases its slot, the next attempt may use another alias.
|
|
pub fn should_reuse_playback_provider(&self, session_owner: &str, provider_name: &Arc<str>) -> bool {
|
|
let session_owner = playback_lease_owner(session_owner);
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
let confirmed = leases
|
|
.lease_of_owner(session_owner)
|
|
.is_some_and(|lease| lease.provider_name == *provider_name && lease.state.is_confirmed());
|
|
drop(leases);
|
|
confirmed || self.has_active_owner_for_provider(provider_name, session_owner)
|
|
}
|
|
|
|
/// True when `session_owner` already backs a live allocation on `provider_name`,
|
|
/// without materialising the full owner set for a membership test.
|
|
fn has_active_owner_for_provider(&self, provider_name: &Arc<str>, session_owner: &str) -> bool {
|
|
let connections = self.read_connections();
|
|
connections.by_owner.get(session_owner).is_some_and(|alloc_ids| {
|
|
alloc_ids.iter().any(|id| {
|
|
if let Some(info) = connections.single.get(id) {
|
|
info.allocation.get_provider_name().as_ref() == Some(provider_name)
|
|
} else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) {
|
|
connections
|
|
.shared
|
|
.by_key
|
|
.get(key)
|
|
.is_some_and(|shared| shared.allocation.get_provider_name().as_ref() == Some(provider_name))
|
|
} else {
|
|
false
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
fn active_reservation_owners(&self, provider_name: &Arc<str>) -> HashSet<Arc<str>> {
|
|
let connections = self.read_connections();
|
|
let mut owners = HashSet::new();
|
|
if let Some(alloc_ids) = connections.by_provider.get(provider_name) {
|
|
for id in alloc_ids {
|
|
if let Some(info) = connections.single.get(id) {
|
|
if let Some(owner) = info.session_owner.as_ref() {
|
|
owners.insert(Arc::clone(owner));
|
|
}
|
|
} else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) {
|
|
if let Some(shared) = connections.shared.by_key.get(key) {
|
|
if let Some(owner) = shared.session_owner.as_ref() {
|
|
owners.insert(Arc::clone(owner));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
for info in connections.single.values() {
|
|
if info.allocation.get_provider_name().as_ref() == Some(provider_name) {
|
|
if let Some(owner) = info.session_owner.as_ref() {
|
|
owners.insert(Arc::clone(owner));
|
|
}
|
|
}
|
|
}
|
|
for shared in connections.shared.by_key.values() {
|
|
if shared.allocation.get_provider_name().as_ref() == Some(provider_name) {
|
|
if let Some(owner) = shared.session_owner.as_ref() {
|
|
owners.insert(Arc::clone(owner));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
owners
|
|
}
|
|
|
|
fn reservation_capacity_usage(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
session_owner: Option<&str>,
|
|
) -> Option<(usize, usize, usize)> {
|
|
let (current_connections, max_connections) = self.providers.provider_capacity(provider_name)?;
|
|
if max_connections == 0 {
|
|
return Some((current_connections, max_connections, 0));
|
|
}
|
|
let counted_owners = self.active_reservation_owners(provider_name);
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
let idle_foreign_reservations = leases.foreign_reserved_slots(provider_name, session_owner, &counted_owners);
|
|
|
|
Some((current_connections, max_connections, idle_foreign_reservations))
|
|
}
|
|
|
|
/// Returns true when the next allocation would consume a slot kept for an idle reservation.
|
|
fn reserved_capacity_blocks_next(&self, provider_name: &Arc<str>, session_owner: Option<&str>) -> bool {
|
|
let Some((current_connections, max_connections, idle_foreign_reservations)) =
|
|
self.reservation_capacity_usage(provider_name, session_owner)
|
|
else {
|
|
return true;
|
|
};
|
|
max_connections > 0
|
|
&& idle_foreign_reservations > 0
|
|
&& current_connections.saturating_add(idle_foreign_reservations) >= max_connections
|
|
}
|
|
|
|
/// Returns true when an already-counted candidate allocation consumed a slot kept for an idle reservation.
|
|
fn exceeds_reserved_capacity(&self, provider_name: &Arc<str>, session_owner: Option<&str>) -> bool {
|
|
let Some((current_connections, max_connections, idle_foreign_reservations)) =
|
|
self.reservation_capacity_usage(provider_name, session_owner)
|
|
else {
|
|
return true;
|
|
};
|
|
max_connections > 0
|
|
&& idle_foreign_reservations > 0
|
|
&& current_connections.saturating_add(idle_foreign_reservations) > max_connections
|
|
}
|
|
|
|
/// Explains a reservation skip with the authoritative slot breakdown instead of
|
|
/// only the transport socket, so the decision can be audited from the logs.
|
|
fn log_reserved_capacity_skip(&self, provider_name: &Arc<str>, params: &AcquireProviderParams<'_>) {
|
|
if !log::log_enabled!(log::Level::Debug) {
|
|
return;
|
|
}
|
|
let Some((current_connections, max_connections, foreign_reserved)) =
|
|
self.reservation_capacity_usage(provider_name, params.session_owner())
|
|
else {
|
|
debug_if_enabled!(
|
|
"Skipping reserved provider {} (reason={}, capacity=unknown, peer_addr={}, request_id={}, playback_kind={})",
|
|
sanitize_sensitive_info(provider_name),
|
|
PlaybackSelectionReason::ReservedCapacity,
|
|
sanitize_sensitive_info(¶ms.addr.to_string()),
|
|
params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()),
|
|
params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string())
|
|
);
|
|
return;
|
|
};
|
|
let usage = self.provider_lease_usage(provider_name);
|
|
debug_if_enabled!(
|
|
"Skipping reserved provider {} (reason={}, current={}, max={}, foreign_reserved={}, active_slots={}, starting_slots={}, idle_slots={}, peer_addr={}, request_id={}, playback_kind={})",
|
|
sanitize_sensitive_info(provider_name),
|
|
PlaybackSelectionReason::ReservedCapacity,
|
|
current_connections,
|
|
max_connections,
|
|
foreign_reserved,
|
|
usage.active,
|
|
usage.starting,
|
|
usage.idle,
|
|
sanitize_sensitive_info(¶ms.addr.to_string()),
|
|
params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()),
|
|
params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string())
|
|
);
|
|
}
|
|
|
|
fn reserved_provider_names_for_other(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
session_owner: Option<&str>,
|
|
) -> HashSet<Arc<str>> {
|
|
let mut reserved = HashSet::new();
|
|
for provider_name in self.providers.provider_names_for_input(input_name) {
|
|
if self.has_foreign_reservation(&provider_name, session_owner) {
|
|
reserved.insert(provider_name);
|
|
}
|
|
}
|
|
reserved
|
|
}
|
|
|
|
pub fn refresh_provider_reservation(&self, provider_name: &Arc<str>, session_owner: &str, ttl_secs: u64) {
|
|
self.refresh_identified_provider_reservation(provider_name, session_owner, PlaybackKind::LiveHls, ttl_secs);
|
|
}
|
|
|
|
/// Recreates or renews a provider reservation for an owner with an explicit kind.
|
|
///
|
|
/// Unlike [`Self::refresh_adaptive_playback_lease`] this recreates the lease when it
|
|
/// no longer exists, which is what a failed preemption needs to restore a victim
|
|
/// reservation that [`Self::clear_identified_provider_reservation`] already removed.
|
|
pub fn refresh_identified_provider_reservation(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
session_owner: &str,
|
|
kind: PlaybackKind,
|
|
ttl_secs: u64,
|
|
) {
|
|
let session_owner = playback_lease_owner(session_owner);
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return;
|
|
}
|
|
let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs);
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
if ttl_secs == 0 {
|
|
leases.release_owner(session_owner);
|
|
return;
|
|
}
|
|
if leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl).is_none() {
|
|
let req_id = PlaybackRequestId::next();
|
|
let _ = leases.begin_owner(session_owner, provider_name, kind, req_id);
|
|
leases.renew_identified_owner(session_owner, provider_name, kind, req_id, granted_ttl);
|
|
}
|
|
}
|
|
|
|
/// Renews the lease of an adaptive (HLS/DASH/Catchup) playback.
|
|
///
|
|
/// Adaptive playback consists of many short requests, so the lease must stay
|
|
/// reconnect-capable: between two segment requests the slot is kept as an idle
|
|
/// lease on the same provider instead of being released and reselected.
|
|
pub fn refresh_adaptive_playback_lease(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
session_owner: &str,
|
|
kind: PlaybackKind,
|
|
ttl_secs: u64,
|
|
) {
|
|
let session_owner = playback_lease_owner(session_owner);
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return;
|
|
}
|
|
let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs);
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl);
|
|
}
|
|
|
|
/// Renews the lease of a playback. A zero TTL clears it.
|
|
pub fn refresh_playback_lease(&self, provider_name: &Arc<str>, lease_ref: &PlaybackLeaseRef<'_>, ttl_secs: u64) {
|
|
let lease_ref = PlaybackLeaseRef { owner: lease_ref.provider_owner(), ..*lease_ref };
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return;
|
|
}
|
|
let granted_ttl = self.renewal_granted_ttl(provider_name, lease_ref.owner, ttl_secs);
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
if ttl_secs == 0 {
|
|
leases.release_identified_owner(lease_ref.owner, lease_ref.request_id);
|
|
return;
|
|
}
|
|
leases.renew_identified_owner(
|
|
lease_ref.owner,
|
|
provider_name,
|
|
lease_ref.kind,
|
|
lease_ref.request_id,
|
|
granted_ttl,
|
|
);
|
|
}
|
|
|
|
/// The reconnect window a renewal may actually grant for `owner`.
|
|
///
|
|
/// A lease that currently holds no reservation right must not gain one through a
|
|
/// mere activity refresh: the grant is re-checked against provider capacity with
|
|
/// the same admission rule as a confirmation, so a previously denied reservation
|
|
/// cannot be silently restored by a later refresh.
|
|
fn renewal_granted_ttl(&self, provider_name: &Arc<str>, owner: &str, requested_ttl_secs: u64) -> u64 {
|
|
if requested_ttl_secs == 0 {
|
|
return 0;
|
|
}
|
|
let already_reserves = {
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
leases.lease_of_owner(owner).is_some_and(|lease| lease.state.is_confirmed() && lease.idle_ttl_secs > 0)
|
|
};
|
|
if already_reserves || self.confirmation_reserve_allowed(provider_name, owner) {
|
|
requested_ttl_secs
|
|
} else {
|
|
0
|
|
}
|
|
}
|
|
|
|
/// Clears an explicit owner. Public HLS tokens intentionally have no owner-wide
|
|
/// delete right: a delayed session cleanup must not erase a retry's shared lease.
|
|
/// HLS request cleanup uses request IDs or binding tags instead.
|
|
pub fn clear_provider_reservation(&self, session_owner: &str) {
|
|
if playback_lease_owner(session_owner) != session_owner {
|
|
return;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut leases = self.write_leases();
|
|
leases.forget_affinity(session_owner);
|
|
leases.release_owner(session_owner);
|
|
}
|
|
|
|
/// Administrative end of the playback behind a session that was actually terminated
|
|
/// (terminate endpoint, kick).
|
|
///
|
|
/// A public HLS token shares its stable owner with the retries of the same
|
|
/// client/user/channel, so it only ends the binding it acquired itself. A stale
|
|
/// token therefore never removes the lease or provider preference of a newer
|
|
/// binding. A bare owner keeps the owner-wide release semantics.
|
|
pub fn terminate_identified_playback_owner(&self, session_token: &str) -> bool {
|
|
let owner = playback_lease_owner(session_token);
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut leases = self.write_leases();
|
|
if owner == session_token {
|
|
leases.forget_affinity(owner);
|
|
return leases.release_owner(owner).is_some();
|
|
}
|
|
leases.terminate_identified_token(owner, session_token)
|
|
}
|
|
|
|
/// Ends the provider preference of a failed playback, but only while it still
|
|
/// belongs to the exact binding that failed. A delayed failure of an older binding
|
|
/// leaves the successor's preference intact.
|
|
pub fn forget_identified_provider_affinity(
|
|
&self,
|
|
session_owner: &str,
|
|
provider_name: &Arc<str>,
|
|
binding_tag: ProviderBindingTag,
|
|
) -> bool {
|
|
let session_owner = playback_lease_owner(session_owner);
|
|
let _transition = self.lock_capacity_transition();
|
|
self.write_leases().forget_identified_affinity(session_owner, provider_name, binding_tag)
|
|
}
|
|
|
|
pub fn clear_identified_provider_reservation(
|
|
&self,
|
|
session_owner: &str,
|
|
provider_name: &Arc<str>,
|
|
binding_tag: Option<ProviderBindingTag>,
|
|
) {
|
|
let session_owner = playback_lease_owner(session_owner);
|
|
// A delayed clear without the exact binding tag has no delete right: it must
|
|
// not fall back to an owner-wide release and erase a successor lease.
|
|
let Some(binding_tag) = binding_tag else {
|
|
return;
|
|
};
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut leases = self.write_leases();
|
|
leases.release_matching_owner(session_owner, provider_name, binding_tag);
|
|
}
|
|
|
|
/// Confirms real media activity for a playback lease. Only a confirmed lease may
|
|
/// outlive its request as a reconnect slot.
|
|
pub fn confirm_playback_activity(&self, owner: &str) -> Option<PlaybackLeaseId> { self.confirm_owner(owner, None) }
|
|
|
|
/// Confirms real media activity for a specific request ID of a playback lease.
|
|
pub fn confirm_identified_playback_activity(
|
|
&self,
|
|
owner: &str,
|
|
request_id: PlaybackRequestId,
|
|
) -> Option<PlaybackLeaseId> {
|
|
self.confirm_owner(owner, Some(request_id))
|
|
}
|
|
|
|
/// Records media activity under the capacity transition. The confirmation only
|
|
/// grants a reservation right when the lease already backs a live allocation or a
|
|
/// free slot remains; otherwise the media bytes are recorded without a reservation
|
|
/// so a late cache confirmation cannot over-commit a provider at its limit.
|
|
fn confirm_owner(&self, owner: &str, request_id: Option<PlaybackRequestId>) -> Option<PlaybackLeaseId> {
|
|
let owner = playback_lease_owner(owner);
|
|
let _transition = self.lock_capacity_transition();
|
|
let (provider_name, wants_reservation) = {
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
let provider_name = leases.provider_for_owner(owner)?;
|
|
let wants_reservation = leases.lease_of_owner(owner).is_some_and(|lease| lease.idle_ttl_secs > 0);
|
|
(provider_name, wants_reservation)
|
|
};
|
|
let reserve = !wants_reservation || self.confirmation_reserve_allowed(&provider_name, owner);
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
match (request_id, reserve) {
|
|
(Some(request_id), true) => leases.confirm_identified_owner(owner, request_id),
|
|
(Some(request_id), false) => leases.confirm_identified_owner_without_reservation(owner, request_id),
|
|
(None, true) => leases.confirm_owner(owner),
|
|
(None, false) => leases.confirm_owner_without_reservation(owner),
|
|
}
|
|
}
|
|
|
|
/// True when confirming `session_owner`'s lease may reserve a provider slot: the
|
|
/// lease either already backs a live allocation, or a free slot remains below the
|
|
/// configured maximum after foreign reservations are counted.
|
|
fn confirmation_reserve_allowed(&self, provider_name: &Arc<str>, session_owner: &str) -> bool {
|
|
if self.has_active_owner_for_provider(provider_name, session_owner) {
|
|
return true;
|
|
}
|
|
let Some((current, max, foreign)) = self.reservation_capacity_usage(provider_name, Some(session_owner)) else {
|
|
return false;
|
|
};
|
|
max == 0 || current.saturating_add(foreign) < max
|
|
}
|
|
|
|
/// Ends a playback request and applies the outcome-specific lease policy.
|
|
///
|
|
/// `idle_ttl_secs` overrides the reconnect window stored on the lease; `None`
|
|
/// keeps the window the endpoint configured when it renewed the lease.
|
|
pub fn finish_playback_request(&self, owner: &str, outcome: PlaybackRequestOutcome, idle_ttl_secs: Option<u64>) {
|
|
self.finish_playback_request_inner(owner, None, outcome, idle_ttl_secs);
|
|
}
|
|
|
|
pub fn finish_identified_playback_request(
|
|
&self,
|
|
owner: &str,
|
|
request_id: PlaybackRequestId,
|
|
outcome: PlaybackRequestOutcome,
|
|
) {
|
|
self.finish_playback_request_inner(owner, Some(request_id), outcome, None);
|
|
}
|
|
|
|
fn finish_playback_request_inner(
|
|
&self,
|
|
owner: &str,
|
|
request_id: Option<PlaybackRequestId>,
|
|
outcome: PlaybackRequestOutcome,
|
|
idle_ttl_secs: Option<u64>,
|
|
) {
|
|
let owner = playback_lease_owner(owner);
|
|
let _transition = self.lock_capacity_transition();
|
|
// A delayed completion must not end a newer request of this playback.
|
|
// Only active connections from a DIFFERENT request ID (or shared streams) prevent lease finish.
|
|
let connections = self.read_connections();
|
|
let has_other_active_connection = connections.by_owner.get(owner).is_some_and(|alloc_ids| {
|
|
alloc_ids.iter().any(|id| {
|
|
connections
|
|
.single
|
|
.get(id)
|
|
.is_some_and(|info| request_id.is_none() || info.playback_request_id != request_id)
|
|
|| connections.shared.shared_by_allocation_id.contains_key(id)
|
|
})
|
|
});
|
|
if has_other_active_connection {
|
|
if let Some(req_id) = request_id {
|
|
let mut leases = self.write_leases();
|
|
leases.detach_identified_request(owner, req_id);
|
|
}
|
|
return;
|
|
}
|
|
drop(connections);
|
|
let mut leases = self.write_leases();
|
|
if let Some(req_id) = request_id {
|
|
leases.finish_identified_owner(owner, req_id, outcome, idle_ttl_secs);
|
|
} else {
|
|
leases.finish_owner(owner, outcome, idle_ttl_secs);
|
|
}
|
|
}
|
|
|
|
/// Snapshot of the logical slots a provider holds, for logs and UI metrics.
|
|
pub fn provider_lease_usage(&self, provider_name: &Arc<str>) -> ProviderLeaseUsage {
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
leases.usage(provider_name)
|
|
}
|
|
|
|
/// Removes every expired starting/idle lease. Runs from the session GC task.
|
|
pub fn prune_expired_leases_now(&self) {
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
}
|
|
|
|
/// The provider lease's binding tag for an owner. The HLS layer captures this
|
|
/// from the acquired handle (via `ProviderHandle::binding_tag`) and passes it
|
|
/// back to [`Self::clear_identified_provider_reservation`] so a stale detach
|
|
/// cannot delete a successor lease on the same account.
|
|
pub fn binding_tag_for_owner(&self, owner: &str) -> Option<ProviderBindingTag> {
|
|
let owner = playback_lease_owner(owner);
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
leases.lease_of_owner(owner).map(|lease| ProviderBindingTag::new(lease.id, lease.binding_generation))
|
|
}
|
|
|
|
fn acquire_exact_connection_inner_no_preempt(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
allow_grace: bool,
|
|
params: &AcquireProviderParams<'_>,
|
|
) -> Option<ProviderHandle> {
|
|
let mut allocation = ProviderAllocationGuard::new(
|
|
self.providers.acquire_exact_connection_with_grace_override(provider_name, allow_grace),
|
|
);
|
|
if matches!(allocation.allocation(), ProviderAllocation::Exhausted) {
|
|
return None;
|
|
}
|
|
if self.exceeds_reserved_capacity(provider_name, params.session_owner()) {
|
|
return None;
|
|
}
|
|
Some(self.register_allocation(allocation.take(), params))
|
|
}
|
|
|
|
fn acquire_exact_connection_inner(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
allow_grace: bool,
|
|
params: &AcquireProviderParams<'_>,
|
|
) -> Option<ProviderHandle> {
|
|
if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, params) {
|
|
return Some(handle);
|
|
}
|
|
if let Some(preempted_alloc) = self.try_preempt_connection(
|
|
provider_name,
|
|
params.priority,
|
|
allow_grace,
|
|
params.kind,
|
|
params.session_owner(),
|
|
) {
|
|
return Some(self.register_allocation(preempted_alloc, params));
|
|
}
|
|
None
|
|
}
|
|
|
|
fn finalize_lineup_allocation(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
allow_grace: bool,
|
|
mut allocation: ProviderAllocationGuard,
|
|
params: &AcquireProviderParams<'_>,
|
|
) -> ProviderHandle {
|
|
if matches!(allocation.allocation(), ProviderAllocation::GracePeriod(_))
|
|
&& self.evict_lower_priority_on_input(input_name, params.priority, params.kind, params.session_owner())
|
|
{
|
|
let evicted_on_same =
|
|
!self.providers.is_over_limit(&allocation.allocation().get_provider_name().unwrap_or_default());
|
|
if !evicted_on_same {
|
|
let mut new_alloc = ProviderAllocationGuard::new(
|
|
self.providers.acquire_connection_with_grace_override(input_name, allow_grace),
|
|
);
|
|
if !matches!(new_alloc.allocation(), ProviderAllocation::Exhausted) {
|
|
if let Some(provider_name) = new_alloc.allocation().get_provider_name() {
|
|
if !self.exceeds_reserved_capacity(&provider_name, params.session_owner()) {
|
|
return self.register_allocation(new_alloc.take(), params);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
self.register_allocation(allocation.take(), params)
|
|
}
|
|
|
|
fn select_victim_from_index(
|
|
&self,
|
|
index: &HashMap<Arc<str>, BTreeMap<PriorityKey, PriorityOwner>>,
|
|
input_name: &Arc<str>,
|
|
requester_priority: Option<i8>,
|
|
reserved_providers: &HashSet<Arc<str>>,
|
|
) -> Option<PreemptionCandidate> {
|
|
let mut victim = None;
|
|
|
|
for (prov_name, tree) in index {
|
|
if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name)
|
|
|| reserved_providers.contains(prov_name)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
let Some(((victim_priority, Reverse(created_at), allocation_id), owner)) = tree.iter().next_back() else {
|
|
continue;
|
|
};
|
|
|
|
if let Some(req_prio) = requester_priority {
|
|
if *victim_priority <= req_prio {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
let candidate = (*owner, *allocation_id, *victim_priority, *created_at);
|
|
if is_better_preemption_candidate(victim, candidate) {
|
|
victim = Some(candidate);
|
|
}
|
|
}
|
|
|
|
victim
|
|
}
|
|
|
|
fn is_normal_preemption_candidate(
|
|
connections: &Connections,
|
|
provider_name: &Arc<str>,
|
|
candidate: PreemptionCandidate,
|
|
) -> bool {
|
|
match candidate.0 {
|
|
PriorityOwner::Single(alloc_id) => connections.single.get(&alloc_id).is_some_and(|info| {
|
|
info.kind == ConnectionKind::Normal
|
|
&& info.lifecycle != ConnectionLifecycle::Closing
|
|
&& info.allocation.get_provider_name().as_ref() == Some(provider_name)
|
|
}),
|
|
PriorityOwner::Shared(shared_id) => connections
|
|
.shared
|
|
.shared_by_allocation_id
|
|
.get(&shared_id)
|
|
.and_then(|key| connections.shared.by_key.get(key))
|
|
.is_some_and(|shared| {
|
|
shared.allocation_id == candidate.1
|
|
&& shared.kind == ConnectionKind::Normal
|
|
&& shared.allocation.get_provider_name().as_ref() == Some(provider_name)
|
|
}),
|
|
}
|
|
}
|
|
|
|
fn select_preemption_candidate(
|
|
&self,
|
|
connections: &Connections,
|
|
input_name: &Arc<str>,
|
|
new_priority: i8,
|
|
kind_needed: ConnectionKind,
|
|
reserved_providers: &HashSet<Arc<str>>,
|
|
) -> Option<PreemptionCandidate> {
|
|
match kind_needed {
|
|
ConnectionKind::Normal => {
|
|
let soft_victim = self.select_victim_from_index(
|
|
&connections.soft_priority_index,
|
|
input_name,
|
|
None,
|
|
reserved_providers,
|
|
);
|
|
if soft_victim.is_some() {
|
|
return soft_victim;
|
|
}
|
|
|
|
let mut victim = None;
|
|
for (prov_name, tree) in &connections.priority_index {
|
|
if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name)
|
|
|| reserved_providers.contains(prov_name)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
let Some(((victim_priority, Reverse(created_at), allocation_id), owner)) = tree.iter().next_back()
|
|
else {
|
|
continue;
|
|
};
|
|
if *victim_priority <= new_priority {
|
|
continue;
|
|
}
|
|
let candidate = (*owner, *allocation_id, *victim_priority, *created_at);
|
|
if !ActiveProviderManager::is_normal_preemption_candidate(connections, prov_name, candidate) {
|
|
continue;
|
|
}
|
|
if is_better_preemption_candidate(victim, candidate) {
|
|
victim = Some(candidate);
|
|
}
|
|
}
|
|
victim
|
|
}
|
|
ConnectionKind::Soft => self.select_victim_from_index(
|
|
&connections.soft_priority_index,
|
|
input_name,
|
|
Some(new_priority),
|
|
reserved_providers,
|
|
),
|
|
}
|
|
}
|
|
|
|
fn acquire_connection_inner_no_preempt(
|
|
&self,
|
|
provider_or_input_name: &Arc<str>,
|
|
allow_grace: bool,
|
|
params: &AcquireProviderParams<'_>,
|
|
) -> Option<ProviderHandle> {
|
|
if let Some(owner) = params.session_owner() {
|
|
match self.owner_provider_pin(provider_or_input_name, owner) {
|
|
Some(OwnerProviderPin::Reserved(reserved_provider)) => {
|
|
return self.acquire_exact_connection_inner_no_preempt(&reserved_provider, allow_grace, params);
|
|
}
|
|
// A continuing playback whose reconnect lease lapsed during a request gap
|
|
// returns to its provider before priority selection, but only into free
|
|
// capacity: the lineup prefers any free slot over a grace over-allocation,
|
|
// so the preference must not force one either.
|
|
Some(OwnerProviderPin::Preferred(affinity_provider)) => {
|
|
if let Some(handle) =
|
|
self.acquire_exact_connection_inner_no_preempt(&affinity_provider, false, params)
|
|
{
|
|
debug_if_enabled!(
|
|
"Provider affinity kept: provider={} owner={} playback_kind={}",
|
|
sanitize_sensitive_info(&affinity_provider),
|
|
sanitize_sensitive_info(owner),
|
|
params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string())
|
|
);
|
|
return Some(handle);
|
|
}
|
|
debug_if_enabled!(
|
|
"Provider affinity unavailable, using lineup: provider={} owner={}",
|
|
sanitize_sensitive_info(&affinity_provider),
|
|
sanitize_sensitive_info(owner)
|
|
);
|
|
}
|
|
None => {}
|
|
}
|
|
}
|
|
|
|
let candidate_count = self.providers.provider_names_for_input(provider_or_input_name).len();
|
|
let attempts = candidate_count.max(1);
|
|
let mut skipped_reserved = HashSet::new();
|
|
for _ in 0..attempts {
|
|
let allocation =
|
|
ProviderAllocationGuard::new(self.providers.acquire_connection_with_grace_override_excluding(
|
|
provider_or_input_name,
|
|
allow_grace,
|
|
&skipped_reserved,
|
|
));
|
|
if matches!(allocation.allocation(), ProviderAllocation::Exhausted) {
|
|
break;
|
|
}
|
|
if let Some(provider_name) = allocation.allocation().get_provider_name() {
|
|
if self.exceeds_reserved_capacity(&provider_name, params.session_owner()) {
|
|
self.log_reserved_capacity_skip(&provider_name, params);
|
|
skipped_reserved.insert(provider_name);
|
|
if skipped_reserved.len() >= attempts {
|
|
break;
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
return Some(self.finalize_lineup_allocation(provider_or_input_name, allow_grace, allocation, params));
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
fn acquire_connection_inner(
|
|
&self,
|
|
provider_or_input_name: &Arc<str>,
|
|
allow_grace: bool,
|
|
params: &AcquireProviderParams<'_>,
|
|
) -> Option<ProviderHandle> {
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
if let Some(handle) = self.acquire_connection_inner_no_preempt(provider_or_input_name, allow_grace, params) {
|
|
return Some(handle);
|
|
}
|
|
|
|
if let Some(preempted_alloc) = self.try_preempt_connection(
|
|
provider_or_input_name,
|
|
params.priority,
|
|
allow_grace,
|
|
params.kind,
|
|
params.session_owner(),
|
|
) {
|
|
return Some(self.register_allocation(preempted_alloc, params));
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
fn register_allocation(
|
|
&self,
|
|
allocation: ProviderAllocation,
|
|
params: &AcquireProviderParams<'_>,
|
|
) -> ProviderHandle {
|
|
let AcquireProviderParams { addr, priority, kind, lease } = *params;
|
|
let session_owner = lease.map(|lease| lease.provider_owner());
|
|
let provider_name = allocation.get_provider_name().unwrap_or_default();
|
|
let allocation_id = self.next_allocation_id.fetch_add(1, Ordering::Relaxed);
|
|
let cancel_token = CancellationToken::new();
|
|
let completion_token = CancellationToken::new();
|
|
let close_reason = Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8));
|
|
let now = Instant::now();
|
|
let is_unlimited = allocation.is_unlimited_provider();
|
|
|
|
// Claim the playback's capacity slot lease in its own lock scope so the
|
|
// connection and lease locks are never nested. The lease starts unconfirmed;
|
|
// it only reserves capacity against other playbacks once real media activity
|
|
// confirms it, which is what keeps abandoned manifest starts from blocking.
|
|
let binding_tag = if let Some(lease) = lease {
|
|
let mut leases = self.write_leases();
|
|
Self::prune_expired_leases(&mut leases);
|
|
let id = leases.begin_owner(lease.provider_owner(), &provider_name, lease.kind, lease.request_id);
|
|
if lease.owner != lease.provider_owner() {
|
|
leases.attach_request_token(id, lease.request_id, lease.owner);
|
|
}
|
|
let generation = leases.lease(id).map_or(1, |lease| lease.binding_generation);
|
|
debug_if_enabled!(
|
|
"Playback lease began: provider={} owner={} kind={} request_id={} lease_id={} generation={} state=starting",
|
|
sanitize_sensitive_info(&provider_name),
|
|
sanitize_sensitive_info(lease.provider_owner()),
|
|
lease.kind,
|
|
lease.request_id,
|
|
id,
|
|
generation
|
|
);
|
|
Some(ProviderBindingTag::new(id, generation))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let session_owner_arc = session_owner.map(Arc::from);
|
|
let mut connections = self.write_connections();
|
|
connections.single.insert(
|
|
allocation_id,
|
|
ActiveConnectionInfo {
|
|
allocation_id,
|
|
client_addr: *addr,
|
|
allocation: allocation.clone(),
|
|
cancel_token: cancel_token.clone(),
|
|
completion_token: completion_token.clone(),
|
|
close_reason: Arc::clone(&close_reason),
|
|
lifecycle: ConnectionLifecycle::Active,
|
|
has_body_owner: false,
|
|
reaper_spawned: false,
|
|
open_generation: 0,
|
|
created_at: now,
|
|
priority,
|
|
kind,
|
|
session_owner: session_owner_arc.clone(),
|
|
request_owner: lease.map(|lease| {
|
|
session_owner_arc
|
|
.as_ref()
|
|
.filter(|owner| owner.as_ref() == lease.owner)
|
|
.map_or_else(|| Arc::from(lease.owner), Arc::clone)
|
|
}),
|
|
playback_request_id: lease.map(|lease| lease.request_id),
|
|
},
|
|
);
|
|
connections.single_by_addr.entry(*addr).or_default().insert(allocation_id);
|
|
Self::index_owner(&mut connections, allocation_id, session_owner_arc.as_ref());
|
|
|
|
// Unlimited providers are not subject to preemption, so we deliberately
|
|
// skip populating the by_provider / priority_index / soft_priority_index
|
|
// indices. The connection itself is still tracked via `single`, so all
|
|
// capacity and lifecycle invariants hold. Without this skip, an
|
|
// unlimited provider's connection can be selected as a preemption victim
|
|
// even though it cannot be exhausted, contradicting the configured
|
|
// `max_connections: 0` semantics.
|
|
if !is_unlimited {
|
|
connections.by_provider.entry(provider_name.clone()).or_default().insert(allocation_id);
|
|
Self::upsert_priority_entry(
|
|
&mut connections,
|
|
&provider_name,
|
|
(priority, Reverse(now), allocation_id),
|
|
PriorityOwner::Single(allocation_id),
|
|
kind,
|
|
);
|
|
}
|
|
|
|
debug_if_enabled!(
|
|
"Added provider connection {provider_name:?} (reason={}, prio={priority}, kind={kind:?}, unlimited={is_unlimited}, allocation_id={allocation_id}, lease_id={}, request_id={}, playback_kind={}, peer_addr={})",
|
|
PlaybackSelectionReason::NewPriorityAllocation,
|
|
binding_tag.map_or_else(String::new, |tag| tag.lease_id.to_string()),
|
|
lease.map_or_else(String::new, |lease| lease.request_id.to_string()),
|
|
lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()),
|
|
sanitize_sensitive_info(&addr.to_string())
|
|
);
|
|
let mut handle = ProviderHandle::new(*addr, allocation_id, allocation, Some(cancel_token));
|
|
handle.completion_token = Some(completion_token);
|
|
handle.close_reason = close_reason;
|
|
handle.playback_request_id = lease.map(|lease| lease.request_id);
|
|
handle.binding_tag = binding_tag;
|
|
handle
|
|
}
|
|
|
|
#[allow(clippy::too_many_lines)]
|
|
/// Evict a single lower-priority connection across the entire input lineup
|
|
/// (all provider aliases). Used when a `GracePeriod` allocation was granted.
|
|
/// Returns true if a victim was successfully evicted.
|
|
fn evict_lower_priority_on_input(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
new_priority: i8,
|
|
kind_needed: ConnectionKind,
|
|
session_owner: Option<&str>,
|
|
) -> bool {
|
|
let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner);
|
|
|
|
let victim = {
|
|
let connections = self.read_connections();
|
|
self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers)
|
|
};
|
|
|
|
let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else {
|
|
return false;
|
|
};
|
|
match owner {
|
|
PriorityOwner::Shared(shared_id) => {
|
|
debug_if_enabled!(
|
|
"Grace-evicting shared connection (allocation_id={shared_id}, prio={v_prio}) on input {} for higher priority request (prio={})",
|
|
sanitize_sensitive_info(input_name),
|
|
new_priority
|
|
);
|
|
|
|
let released = {
|
|
let mut connections = self.write_connections();
|
|
let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else {
|
|
return false;
|
|
};
|
|
|
|
let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| {
|
|
shared.allocation_id == alloc_id
|
|
&& shared.priority == v_prio
|
|
&& shared.created_at == victim_created_at
|
|
});
|
|
if !still_match {
|
|
return false;
|
|
}
|
|
|
|
if let Some(shared) = connections.shared.by_key.remove(&key) {
|
|
connections.shared.shared_by_allocation_id.remove(&shared.allocation_id);
|
|
Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref());
|
|
for subscriber_id in shared.connections.keys() {
|
|
connections.shared.key_by_subscriber.remove(subscriber_id);
|
|
}
|
|
if let Some(name) = shared.allocation.get_provider_name() {
|
|
if let Some(list) = connections.by_provider.get_mut(&name) {
|
|
list.remove(&shared.allocation_id);
|
|
}
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
&name,
|
|
&(v_prio, Reverse(victim_created_at), alloc_id),
|
|
shared.kind,
|
|
);
|
|
}
|
|
Some((key, shared.allocation, shared.cancel_token))
|
|
} else {
|
|
return false;
|
|
}
|
|
};
|
|
if let Some((stream_url, allocation, cancel_token)) = released {
|
|
if let Some(token) = cancel_token {
|
|
token.cancel();
|
|
}
|
|
if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) {
|
|
tokio::spawn(async move {
|
|
ssm.teardown_preempted_stream(&stream_url, alloc_id).await;
|
|
allocation.release();
|
|
});
|
|
return false;
|
|
}
|
|
allocation.release();
|
|
}
|
|
}
|
|
PriorityOwner::Single(victim_alloc_id) => {
|
|
let mut connections = self.write_connections();
|
|
if let Some(info) = connections.single.get(&victim_alloc_id) {
|
|
if info.priority != v_prio || info.created_at != victim_created_at {
|
|
return false;
|
|
}
|
|
}
|
|
if let Some(info) = connections.single.get_mut(&victim_alloc_id) {
|
|
debug_if_enabled!(
|
|
"Grace-evicting single connection from {} (prio={}) on input {} for higher priority request (prio={})",
|
|
sanitize_sensitive_info(&info.client_addr.to_string()),
|
|
v_prio,
|
|
sanitize_sensitive_info(input_name),
|
|
new_priority
|
|
);
|
|
info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release);
|
|
}
|
|
let action = ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id);
|
|
if let ReleaseAction::Wait(alloc_id, gen, completion_token) = action {
|
|
let shutdown_token = self.shutdown_token.clone();
|
|
let core = Arc::clone(&self.core);
|
|
tokio::spawn(async move {
|
|
tokio::select! {
|
|
() = shutdown_token.cancelled() => {},
|
|
() = completion_token.cancelled() => {
|
|
core.complete_release_with_generation(alloc_id, Some(gen));
|
|
}
|
|
}
|
|
});
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
true
|
|
}
|
|
|
|
fn try_acquire_allocation_after_freed(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
allow_grace: bool,
|
|
reserved_providers: &HashSet<Arc<str>>,
|
|
session_owner: Option<&str>,
|
|
) -> Option<ProviderAllocation> {
|
|
let attempts = self.providers.provider_names_for_input(input_name).len().max(1);
|
|
let mut excluded_providers = reserved_providers.clone();
|
|
for _ in 0..attempts {
|
|
let allocation = self.providers.acquire_connection_with_grace_override_excluding(
|
|
input_name,
|
|
allow_grace,
|
|
&excluded_providers,
|
|
);
|
|
if matches!(allocation, ProviderAllocation::Exhausted) {
|
|
break;
|
|
}
|
|
if let Some(provider_name) = allocation.get_provider_name() {
|
|
if self.exceeds_reserved_capacity(&provider_name, session_owner) {
|
|
excluded_providers.insert(provider_name);
|
|
allocation.release();
|
|
continue;
|
|
}
|
|
}
|
|
return Some(allocation);
|
|
}
|
|
None
|
|
}
|
|
|
|
#[allow(clippy::too_many_lines)]
|
|
fn try_preempt_connection_outcome(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
new_priority: i8,
|
|
allow_grace: bool,
|
|
kind_needed: ConnectionKind,
|
|
session_owner: Option<&str>,
|
|
) -> PreemptionOutcome {
|
|
let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner);
|
|
let victim = {
|
|
let connections = self.read_connections();
|
|
self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers)
|
|
};
|
|
|
|
let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else {
|
|
return PreemptionOutcome::Exhausted;
|
|
};
|
|
|
|
match owner {
|
|
PriorityOwner::Shared(shared_id) => {
|
|
debug_if_enabled!(
|
|
"Preempting shared connection (allocation_id={shared_id}, prio={v_prio}) for higher priority request (prio={new_priority})"
|
|
);
|
|
let released_shared_allocation = {
|
|
let mut connections = self.write_connections();
|
|
let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else {
|
|
return PreemptionOutcome::Exhausted;
|
|
};
|
|
|
|
let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| {
|
|
shared.allocation_id == alloc_id
|
|
&& shared.priority == v_prio
|
|
&& shared.created_at == victim_created_at
|
|
});
|
|
if !still_match {
|
|
return PreemptionOutcome::Exhausted;
|
|
}
|
|
|
|
if let Some(shared) = connections.shared.by_key.remove(&key) {
|
|
connections.shared.shared_by_allocation_id.remove(&shared.allocation_id);
|
|
Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref());
|
|
for subscriber_id in shared.connections.keys() {
|
|
connections.shared.key_by_subscriber.remove(subscriber_id);
|
|
}
|
|
|
|
if let Some(name) = shared.allocation.get_provider_name() {
|
|
if let Some(list) = connections.by_provider.get_mut(&name) {
|
|
list.remove(&shared.allocation_id);
|
|
}
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
&name,
|
|
&(v_prio, Reverse(victim_created_at), alloc_id),
|
|
shared.kind,
|
|
);
|
|
}
|
|
Some((key, shared.allocation, shared.cancel_token))
|
|
} else {
|
|
None
|
|
}
|
|
};
|
|
|
|
let Some((stream_url, allocation, cancel_token)) = released_shared_allocation else {
|
|
return PreemptionOutcome::Exhausted;
|
|
};
|
|
|
|
if let Some(token) = cancel_token {
|
|
token.cancel();
|
|
}
|
|
if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) {
|
|
// Signal completion after teardown finishes so async acquirers can wait for the
|
|
// actual upstream shutdown rather than receiving Exhausted immediately.
|
|
let done_token = CancellationToken::new();
|
|
let done_signal = done_token.clone();
|
|
tokio::spawn(async move {
|
|
ssm.teardown_preempted_stream(&stream_url, alloc_id).await;
|
|
allocation.release();
|
|
done_signal.cancel();
|
|
});
|
|
PreemptionOutcome::PendingCompletion(None, done_token)
|
|
} else {
|
|
allocation.release();
|
|
if let Some(alloc) = self.try_acquire_allocation_after_freed(
|
|
input_name,
|
|
allow_grace,
|
|
&reserved_providers,
|
|
session_owner,
|
|
) {
|
|
PreemptionOutcome::Acquired(alloc)
|
|
} else {
|
|
PreemptionOutcome::Exhausted
|
|
}
|
|
}
|
|
}
|
|
PriorityOwner::Single(victim_alloc_id) => {
|
|
let (action, completion_token) = {
|
|
let mut connections = self.write_connections();
|
|
if let Some(info) = connections.single.get(&victim_alloc_id) {
|
|
if info.priority != v_prio || info.created_at != victim_created_at {
|
|
return PreemptionOutcome::Exhausted;
|
|
}
|
|
} else {
|
|
return PreemptionOutcome::Exhausted;
|
|
}
|
|
if let Some(info) = connections.single.get_mut(&victim_alloc_id) {
|
|
debug_if_enabled!(
|
|
"Preempting single connection from {} (prio={v_prio}) for higher priority request (prio={new_priority})",
|
|
sanitize_sensitive_info(&info.client_addr.to_string())
|
|
);
|
|
info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release);
|
|
}
|
|
let completion_token =
|
|
connections.single.get(&victim_alloc_id).map(|info| info.completion_token.clone());
|
|
let action =
|
|
ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id);
|
|
(action, completion_token)
|
|
};
|
|
|
|
match action {
|
|
ReleaseAction::Wait(victim_id, gen, comp_token) => {
|
|
let shutdown_token = self.shutdown_token.clone();
|
|
let core = Arc::clone(&self.core);
|
|
let reaper_token = comp_token.clone();
|
|
tokio::spawn(async move {
|
|
tokio::select! {
|
|
() = shutdown_token.cancelled() => {},
|
|
() = reaper_token.cancelled() => {
|
|
core.complete_release_with_generation(victim_id, Some(gen));
|
|
}
|
|
}
|
|
});
|
|
PreemptionOutcome::PendingCompletion(Some((victim_id, gen)), comp_token)
|
|
}
|
|
ReleaseAction::None => {
|
|
if let Some(token) = completion_token {
|
|
// Retrieve the current generation for the idempotent self-release path.
|
|
let victim_identity = {
|
|
let connections = self.read_connections();
|
|
connections
|
|
.single
|
|
.get(&victim_alloc_id)
|
|
.map(|info| (victim_alloc_id, info.open_generation))
|
|
};
|
|
PreemptionOutcome::PendingCompletion(victim_identity, token)
|
|
} else {
|
|
PreemptionOutcome::Exhausted
|
|
}
|
|
}
|
|
ReleaseAction::Immediate => {
|
|
if let Some(alloc) = self.try_acquire_allocation_after_freed(
|
|
input_name,
|
|
allow_grace,
|
|
&reserved_providers,
|
|
session_owner,
|
|
) {
|
|
PreemptionOutcome::Acquired(alloc)
|
|
} else {
|
|
PreemptionOutcome::Exhausted
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn try_preempt_connection(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
new_priority: i8,
|
|
allow_grace: bool,
|
|
kind_needed: ConnectionKind,
|
|
session_owner: Option<&str>,
|
|
) -> Option<ProviderAllocation> {
|
|
match self.try_preempt_connection_outcome(input_name, new_priority, allow_grace, kind_needed, session_owner) {
|
|
PreemptionOutcome::Acquired(alloc) => Some(alloc),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
pub fn acquire_exact_connection_with_grace(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
) -> Option<ProviderHandle> {
|
|
self.acquire_exact_connection_with_grace_for_session(provider_name, addr, allow_grace, priority, kind, None)
|
|
}
|
|
|
|
pub fn acquire_exact_connection_with_grace_for_session(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
session_owner: Option<&str>,
|
|
) -> Option<ProviderHandle> {
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
let lease = session_owner.map(PlaybackLeaseRef::for_session_owner);
|
|
self.acquire_exact_connection_inner(
|
|
provider_name,
|
|
allow_grace,
|
|
&AcquireProviderParams { addr, priority, kind, lease: lease.as_ref().copied() },
|
|
)
|
|
}
|
|
|
|
pub fn force_exact_acquire_connection(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
) -> Option<ProviderHandle> {
|
|
// Compatibility wrapper: keep the exact-provider behavior but do not over-allocate exhausted accounts.
|
|
self.acquire_exact_connection_with_grace(provider_name, addr, false, priority, kind)
|
|
}
|
|
|
|
// Returns the next available provider connection
|
|
pub fn acquire_connection(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
) -> Option<ProviderHandle> {
|
|
self.acquire_connection_inner(input_name, true, &AcquireProviderParams { addr, priority, kind, lease: None })
|
|
}
|
|
|
|
/// Acquire a provider connection while explicitly controlling provider-side grace allocations.
|
|
pub fn acquire_connection_with_grace(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
) -> Option<ProviderHandle> {
|
|
self.acquire_connection_with_grace_for_session(input_name, addr, allow_grace, priority, kind, None)
|
|
}
|
|
|
|
pub fn acquire_connection_with_grace_for_session(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
session_owner: Option<&str>,
|
|
) -> Option<ProviderHandle> {
|
|
self.acquire_connection_with_lease_for_session(
|
|
input_name,
|
|
addr,
|
|
allow_grace,
|
|
priority,
|
|
kind,
|
|
session_owner.map(PlaybackLeaseRef::for_session_owner),
|
|
)
|
|
}
|
|
|
|
/// Lineup acquisition with an explicit playback lease identity.
|
|
pub fn acquire_connection_with_lease_for_session(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
lease: Option<PlaybackLeaseRef<'_>>,
|
|
) -> Option<ProviderHandle> {
|
|
self.acquire_connection_inner(input_name, allow_grace, &AcquireProviderParams { addr, priority, kind, lease })
|
|
}
|
|
|
|
/// Lineup acquisition with an explicit playback lease identity, awaiting preemption completion if needed.
|
|
pub async fn acquire_connection_with_lease_for_session_await(
|
|
&self,
|
|
input_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
lease: Option<PlaybackLeaseRef<'_>>,
|
|
) -> Option<ProviderHandle> {
|
|
let params = AcquireProviderParams { addr, priority, kind, lease };
|
|
let outcome = {
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
if let Some(handle) = self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) {
|
|
return Some(handle);
|
|
}
|
|
self.try_preempt_connection_outcome(
|
|
input_name,
|
|
params.priority,
|
|
allow_grace,
|
|
params.kind,
|
|
params.session_owner(),
|
|
)
|
|
};
|
|
|
|
match outcome {
|
|
PreemptionOutcome::Acquired(alloc) => {
|
|
let _transition = self.lock_capacity_transition();
|
|
Some(self.register_allocation(alloc, ¶ms))
|
|
}
|
|
PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => {
|
|
let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await;
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
// Idempotently free the victim's slot in case the background reaper has not yet run.
|
|
// complete_release_locked is a no-op if the slot was already freed or the generation
|
|
// no longer matches.
|
|
if let Some((victim_alloc_id, victim_gen)) = victim_identity {
|
|
let mut connections = self.write_connections();
|
|
ActiveProviderManagerCore::complete_release_locked(
|
|
&mut connections,
|
|
victim_alloc_id,
|
|
Some(victim_gen),
|
|
);
|
|
}
|
|
self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms)
|
|
}
|
|
PreemptionOutcome::Exhausted => None,
|
|
}
|
|
}
|
|
|
|
/// Exact-provider acquisition with an explicit playback lease identity.
|
|
pub fn acquire_exact_connection_with_lease_for_session(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
lease: Option<PlaybackLeaseRef<'_>>,
|
|
) -> Option<ProviderHandle> {
|
|
let _transition = self.lock_capacity_transition();
|
|
self.acquire_exact_connection_inner(
|
|
provider_name,
|
|
allow_grace,
|
|
&AcquireProviderParams { addr, priority, kind, lease },
|
|
)
|
|
}
|
|
|
|
/// Exact-provider acquisition with an explicit playback lease identity, awaiting preemption completion if needed.
|
|
pub async fn acquire_exact_connection_with_lease_for_session_await(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
addr: &SocketAddr,
|
|
allow_grace: bool,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
lease: Option<PlaybackLeaseRef<'_>>,
|
|
) -> Option<ProviderHandle> {
|
|
let params = AcquireProviderParams { addr, priority, kind, lease };
|
|
let outcome = {
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) {
|
|
return Some(handle);
|
|
}
|
|
self.try_preempt_connection_outcome(
|
|
provider_name,
|
|
params.priority,
|
|
allow_grace,
|
|
params.kind,
|
|
params.session_owner(),
|
|
)
|
|
};
|
|
|
|
match outcome {
|
|
PreemptionOutcome::Acquired(alloc) => {
|
|
let _transition = self.lock_capacity_transition();
|
|
Some(self.register_allocation(alloc, ¶ms))
|
|
}
|
|
PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => {
|
|
let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await;
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return None;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
if let Some((victim_alloc_id, victim_gen)) = victim_identity {
|
|
let mut connections = self.write_connections();
|
|
ActiveProviderManagerCore::complete_release_locked(
|
|
&mut connections,
|
|
victim_alloc_id,
|
|
Some(victim_gen),
|
|
);
|
|
}
|
|
self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms)
|
|
}
|
|
PreemptionOutcome::Exhausted => None,
|
|
}
|
|
}
|
|
|
|
/// Acquire a provider connection for probe tasks with configurable priority.
|
|
/// Probes never consume grace capacity.
|
|
pub fn acquire_connection_for_probe(&self, input_name: &Arc<str>, priority: i8) -> Option<ProviderHandle> {
|
|
self.acquire_connection_inner(
|
|
input_name,
|
|
false,
|
|
&AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None },
|
|
)
|
|
}
|
|
|
|
/// Acquire a provider connection for background transfers (downloads/recordings).
|
|
/// Transfers participate in the same provider priority/preemption model as normal
|
|
/// streams, but they never consume grace capacity and wait externally on notifications.
|
|
pub fn acquire_connection_for_download(&self, input_name: &Arc<str>, priority: i8) -> Option<ProviderHandle> {
|
|
self.acquire_connection_inner(
|
|
input_name,
|
|
false,
|
|
&AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None },
|
|
)
|
|
}
|
|
|
|
// This method is used for redirects to cycle through the provider
|
|
pub fn get_next_provider(&self, provider_name: &Arc<str>) -> Option<Arc<ProviderConfig>> {
|
|
self.providers.get_next_provider(provider_name)
|
|
}
|
|
|
|
pub fn find_provider_config(&self, provider_name: &Arc<str>) -> Option<Arc<ProviderConfig>> {
|
|
self.providers.find_provider_config(provider_name)
|
|
}
|
|
|
|
pub fn is_provider_for_input(&self, provider_name: &Arc<str>, input_name: &Arc<str>) -> bool {
|
|
self.providers.is_provider_for_input(provider_name.as_ref(), input_name.as_ref())
|
|
}
|
|
|
|
pub fn is_provider_reserved_for_other_session(
|
|
&self,
|
|
provider_name: &Arc<str>,
|
|
session_owner: Option<&str>,
|
|
) -> bool {
|
|
let session_owner = session_owner.map(playback_lease_owner);
|
|
self.reserved_capacity_blocks_next(provider_name, session_owner)
|
|
}
|
|
|
|
pub fn active_connections(&self) -> Option<HashMap<Arc<str>, usize>> { self.providers.active_connections() }
|
|
|
|
pub fn is_over_limit(&self, provider_name: &Arc<str>) -> bool { self.providers.is_over_limit(provider_name) }
|
|
|
|
pub fn is_exhausted(&self, provider_name: &Arc<str>) -> bool { self.providers.is_exhausted(provider_name) }
|
|
|
|
/// Finds every shared subscription carried by one transport socket.
|
|
///
|
|
/// This is an explicit socket-wide transport action (connection close, kick), not
|
|
/// the normal playback cleanup path. A single shared subscription is released
|
|
/// through `release_shared_subscriber`, which cannot touch other clients that
|
|
/// happen to arrive through the same reverse-proxy socket.
|
|
fn release_shared_by_addr(&self, addr: &SocketAddr) -> Vec<SharedSubscriberId> {
|
|
let connections = self.read_connections();
|
|
connections
|
|
.shared
|
|
.by_key
|
|
.values()
|
|
.flat_map(|shared| {
|
|
shared.connections.iter().filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Releases exactly one shared subscription and rebalances the shared allocation.
|
|
///
|
|
/// Returns the allocation to release when the last subscriber left.
|
|
fn release_shared_subscriber(&self, subscriber_id: SharedSubscriberId) -> Option<ProviderAllocation> {
|
|
let mut connections = self.write_connections();
|
|
|
|
let key = connections.shared.key_by_subscriber.remove(&subscriber_id)?;
|
|
let mut shared = connections.shared.by_key.remove(&key)?;
|
|
shared.connections.remove(&subscriber_id);
|
|
|
|
let shared_is_unlimited = shared.allocation.is_unlimited_provider();
|
|
|
|
if shared.connections.is_empty() {
|
|
connections.shared.shared_by_allocation_id.remove(&shared.allocation_id);
|
|
Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref());
|
|
if !shared_is_unlimited {
|
|
if let Some(name) = shared.allocation.get_provider_name() {
|
|
if let Some(list) = connections.by_provider.get_mut(&name) {
|
|
list.remove(&shared.allocation_id);
|
|
}
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
&name,
|
|
&(shared.priority, Reverse(shared.created_at), shared.allocation_id),
|
|
shared.kind,
|
|
);
|
|
}
|
|
}
|
|
return Some(shared.allocation);
|
|
}
|
|
|
|
// Recompute shared priority from remaining subscribers so preemption decisions
|
|
// reflect who is actually still watching the shared stream. Unlimited shared
|
|
// streams are not in the preemption index, so this rebalance is skipped.
|
|
if !shared_is_unlimited {
|
|
let old_priority = shared.priority;
|
|
let old_kind = shared.kind;
|
|
shared.kind = Self::shared_effective_kind(&shared.connections);
|
|
if let Some(new_priority) = Self::shared_effective_priority(&shared.connections, shared.kind) {
|
|
shared.priority = new_priority;
|
|
if (new_priority, shared.kind) != (old_priority, old_kind) {
|
|
if let Some(name) = shared.allocation.get_provider_name() {
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
&name,
|
|
&(old_priority, Reverse(shared.created_at), shared.allocation_id),
|
|
old_kind,
|
|
);
|
|
Self::upsert_priority_entry(
|
|
&mut connections,
|
|
&name,
|
|
(new_priority, Reverse(shared.created_at), shared.allocation_id),
|
|
PriorityOwner::Shared(shared.allocation_id),
|
|
shared.kind,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
connections.shared.by_key.insert(key, shared);
|
|
None
|
|
}
|
|
|
|
pub fn release_shared_connection(&self, subscriber_id: SharedSubscriberId) {
|
|
let _transition = self.lock_capacity_transition();
|
|
self.release_shared_connection_inner(subscriber_id);
|
|
}
|
|
|
|
fn release_shared_connection_inner(&self, subscriber_id: SharedSubscriberId) {
|
|
if let Some(allocation) = self.release_shared_subscriber(subscriber_id) {
|
|
debug_if_enabled!(
|
|
"Released last shared connection for provider {} (subscriber={subscriber_id})",
|
|
allocation.get_provider_name().unwrap_or_default()
|
|
);
|
|
allocation.release();
|
|
}
|
|
}
|
|
|
|
pub fn register_body_owner(&self, alloc_id: AllocationId) -> bool {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
if let Some(info) = connections.single.get_mut(&alloc_id) {
|
|
if info.lifecycle == ConnectionLifecycle::Closing || info.lifecycle == ConnectionLifecycle::Closed {
|
|
return false;
|
|
}
|
|
info.has_body_owner = true;
|
|
if info.lifecycle == ConnectionLifecycle::Opening {
|
|
info.lifecycle = ConnectionLifecycle::Active;
|
|
}
|
|
true
|
|
} else {
|
|
false
|
|
}
|
|
}
|
|
|
|
pub fn mark_opening(&self, alloc_id: AllocationId) {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
if let Some(info) = connections.single.get_mut(&alloc_id) {
|
|
if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed {
|
|
info.lifecycle = ConnectionLifecycle::Opening;
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn renew_opening_tokens(&self, alloc_id: AllocationId) -> Option<(CancellationToken, CancellationToken, u64)> {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
if let Some(info) = connections.single.get_mut(&alloc_id) {
|
|
if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed {
|
|
info.lifecycle = ConnectionLifecycle::Opening;
|
|
info.open_generation = info.open_generation.wrapping_add(1);
|
|
info.reaper_spawned = false;
|
|
let cancel_token = CancellationToken::new();
|
|
let completion_token = CancellationToken::new();
|
|
info.cancel_token = cancel_token.clone();
|
|
info.completion_token = completion_token.clone();
|
|
return Some((cancel_token, completion_token, info.open_generation));
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
pub fn mark_closing(&self, alloc_id: AllocationId) {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
let prio = if let Some(info) = connections.single.get_mut(&alloc_id) {
|
|
info.lifecycle = ConnectionLifecycle::Closing;
|
|
info.cancel_token.cancel();
|
|
info.allocation
|
|
.get_provider_name()
|
|
.map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind))
|
|
} else {
|
|
None
|
|
};
|
|
if let Some((name, key, kind)) = prio {
|
|
Self::remove_priority_entry(&mut connections, &name, &key, kind);
|
|
}
|
|
}
|
|
|
|
pub fn release_connection(&self, addr: &SocketAddr) {
|
|
let _transition = self.lock_capacity_transition();
|
|
let single_alloc_ids = {
|
|
let mut connections = self.write_connections();
|
|
connections.single_by_addr.remove(addr)
|
|
};
|
|
|
|
if let Some(alloc_ids) = single_alloc_ids {
|
|
let mut to_reap = Vec::new();
|
|
{
|
|
let mut connections = self.write_connections();
|
|
for id in alloc_ids {
|
|
if let ReleaseAction::Wait(alloc_id, gen, token) =
|
|
ActiveProviderManagerCore::plan_single_release_locked(&mut connections, id)
|
|
{
|
|
to_reap.push((alloc_id, gen, token));
|
|
}
|
|
}
|
|
}
|
|
for (id, gen, completion_token) in to_reap {
|
|
let core = Arc::clone(&self.core);
|
|
let shutdown_token = self.shutdown_token.clone();
|
|
if let Ok(rt_handle) = tokio::runtime::Handle::try_current() {
|
|
rt_handle.spawn(async move {
|
|
tokio::select! {
|
|
() = shutdown_token.cancelled() => {},
|
|
() = completion_token.cancelled() => {
|
|
core.complete_release_with_generation(id, Some(gen));
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Shared connections carried by this transport socket. Resolve the subscriber
|
|
// ids first, then release each one precisely; two external clients behind one
|
|
// reverse proxy hold distinct subscriber ids and never overwrite each other.
|
|
for subscriber_id in self.release_shared_by_addr(addr) {
|
|
self.release_shared_connection_inner(subscriber_id);
|
|
}
|
|
}
|
|
|
|
fn release_shared_handle_locked(
|
|
connections: &mut Connections,
|
|
handle: &ProviderHandle,
|
|
) -> Option<ProviderAllocation> {
|
|
let mut released = None;
|
|
let mut released_priority_key: Option<(Arc<str>, PriorityKey, ConnectionKind)> = None;
|
|
if let Some(key) = connections.shared.shared_by_allocation_id.remove(&handle.allocation_id) {
|
|
if let Some(shared) = connections.shared.by_key.remove(&key) {
|
|
Self::unindex_owner(connections, handle.allocation_id, shared.session_owner.as_ref());
|
|
let pkey = (shared.priority, Reverse(shared.created_at), handle.allocation_id);
|
|
let shared_kind = shared.kind;
|
|
let shared_is_unlimited = shared.allocation.is_unlimited_provider();
|
|
released = Some(shared.allocation);
|
|
for subscriber_id in shared.connections.keys() {
|
|
connections.shared.key_by_subscriber.remove(subscriber_id);
|
|
}
|
|
if !shared_is_unlimited {
|
|
if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) {
|
|
if let Some(list) = connections.by_provider.get_mut(&name) {
|
|
list.remove(&handle.allocation_id);
|
|
}
|
|
released_priority_key = Some((name, pkey, shared_kind));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if let Some((name, pkey, kind)) = &released_priority_key {
|
|
Self::remove_priority_entry(connections, name, pkey, *kind);
|
|
}
|
|
if let Some(allocation) = released.as_ref() {
|
|
allocation.release();
|
|
}
|
|
released
|
|
}
|
|
|
|
pub fn release_handle(&self, handle: &ProviderHandle) {
|
|
let _transition = self.lock_capacity_transition();
|
|
let wait_action = {
|
|
let mut connections = self.write_connections();
|
|
if connections.single.contains_key(&handle.allocation_id) {
|
|
if handle.completion_token.as_ref().is_some_and(CancellationToken::is_cancelled) {
|
|
if let Some(info) = connections.single.get_mut(&handle.allocation_id) {
|
|
if info.open_generation == handle.open_generation {
|
|
info.completion_token.cancel();
|
|
}
|
|
}
|
|
}
|
|
ActiveProviderManagerCore::plan_single_release_locked(&mut connections, handle.allocation_id)
|
|
} else {
|
|
Self::release_shared_handle_locked(&mut connections, handle);
|
|
ReleaseAction::None
|
|
}
|
|
};
|
|
|
|
if let ReleaseAction::Wait(alloc_id, gen, completion_token) = wait_action {
|
|
let core = Arc::clone(&self.core);
|
|
let shutdown_token = self.shutdown_token.clone();
|
|
if let Ok(rt_handle) = tokio::runtime::Handle::try_current() {
|
|
rt_handle.spawn(async move {
|
|
tokio::select! {
|
|
() = shutdown_token.cancelled() => {},
|
|
() = completion_token.cancelled() => {
|
|
core.complete_release_with_generation(alloc_id, Some(gen));
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Stops only stale requests of this playback, even on a multiplexed proxy socket.
|
|
pub fn release_playback_connections(&self, owner: &str, addrs: &[SocketAddr]) {
|
|
let handles = {
|
|
let connections = self.read_connections();
|
|
let mut handles = Vec::new();
|
|
for addr in addrs {
|
|
if let Some(alloc_ids) = connections.single_by_addr.get(addr) {
|
|
for id in alloc_ids {
|
|
if let Some(info) = connections.single.get(id) {
|
|
if info.request_owner.as_deref() == Some(owner) {
|
|
handles.push(ProviderHandle {
|
|
playback_request_id: info.playback_request_id,
|
|
binding_tag: None,
|
|
client_id: info.client_addr,
|
|
allocation_id: info.allocation_id,
|
|
allocation: info.allocation.clone(),
|
|
cancel_token: Some(info.cancel_token.clone()),
|
|
completion_token: Some(info.completion_token.clone()),
|
|
close_reason: Arc::clone(&info.close_reason),
|
|
open_generation: info.open_generation,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
handles
|
|
};
|
|
for handle in handles {
|
|
handle.set_close_reason(ProviderCloseReason::Superseded);
|
|
if let Some(token) = &handle.cancel_token {
|
|
token.cancel();
|
|
}
|
|
self.release_handle(&handle);
|
|
}
|
|
}
|
|
|
|
/// Stops stale requests of this playback and awaits upstream body closure before releasing capacity.
|
|
pub async fn release_playback_connections_await(&self, owner: &str, addrs: &[SocketAddr]) {
|
|
let targets = {
|
|
let mut connections = self.write_connections();
|
|
let mut targets = Vec::new();
|
|
for addr in addrs {
|
|
if let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() {
|
|
for id in alloc_ids {
|
|
if let Some(info) = connections.single.get_mut(&id) {
|
|
if info.request_owner.as_deref() == Some(owner)
|
|
&& info.lifecycle != ConnectionLifecycle::Closed
|
|
{
|
|
info.lifecycle = ConnectionLifecycle::Closing;
|
|
info.close_reason.store(ProviderCloseReason::Superseded as u8, Ordering::Release);
|
|
targets.push((
|
|
info.allocation_id,
|
|
info.cancel_token.clone(),
|
|
info.completion_token.clone(),
|
|
info.priority,
|
|
info.created_at,
|
|
info.kind,
|
|
info.allocation.get_provider_name(),
|
|
info.open_generation,
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (alloc_id, _, _, priority, created_at, kind, provider_name, _) in &targets {
|
|
if let Some(name) = provider_name {
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
name,
|
|
&(*priority, Reverse(*created_at), *alloc_id),
|
|
*kind,
|
|
);
|
|
}
|
|
}
|
|
targets
|
|
};
|
|
|
|
if targets.is_empty() {
|
|
return;
|
|
}
|
|
|
|
for (_, cancel_token, _, _, _, _, _, _) in &targets {
|
|
cancel_token.cancel();
|
|
}
|
|
|
|
let futures: Vec<_> = targets.iter().map(|(_, _, token, _, _, _, _, _)| token.cancelled()).collect();
|
|
let _ = tokio::time::timeout(Duration::from_secs(1), futures::future::join_all(futures)).await;
|
|
|
|
for (alloc_id, _, completion_token, _, _, _, _, gen) in targets {
|
|
if completion_token.is_cancelled() {
|
|
self.core.complete_release_with_generation(alloc_id, Some(gen));
|
|
} else {
|
|
log::warn!("Superseded provider connection {alloc_id} did not close within deadline, retaining slot as Closing until reaped");
|
|
// Atomically claim the reaper role under the lock before spawning, so that
|
|
// if this future was dropped between collection and here, release_handle can
|
|
// still install its own reaper (reaper_spawned remains false until now).
|
|
let should_spawn = {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
connections.single.get_mut(&alloc_id).is_some_and(|info| {
|
|
if info.reaper_spawned {
|
|
false
|
|
} else {
|
|
info.reaper_spawned = true;
|
|
true
|
|
}
|
|
})
|
|
};
|
|
if should_spawn {
|
|
let core = Arc::clone(&self.core);
|
|
let shutdown_token = self.shutdown_token.clone();
|
|
tokio::spawn(async move {
|
|
tokio::select! {
|
|
() = shutdown_token.cancelled() => {},
|
|
() = completion_token.cancelled() => {
|
|
core.complete_release_with_generation(alloc_id, Some(gen));
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn reclassify_connection(&self, addr: &SocketAddr, kind: ConnectionKind, priority: i8) -> bool {
|
|
self.reclassify_connection_for_owner(addr, None, kind, priority)
|
|
}
|
|
|
|
pub fn reclassify_connection_for_owner(
|
|
&self,
|
|
addr: &SocketAddr,
|
|
owner: Option<&str>,
|
|
kind: ConnectionKind,
|
|
priority: i8,
|
|
) -> bool {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
|
|
let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() else {
|
|
return false;
|
|
};
|
|
let mut index_updates = Vec::new();
|
|
for allocation_id in alloc_ids {
|
|
if let Some(info) = connections.single.get_mut(&allocation_id) {
|
|
if info.request_owner.as_deref() != owner {
|
|
continue;
|
|
}
|
|
if let Some(provider_name) = info.allocation.get_provider_name() {
|
|
let old_key = (info.priority, Reverse(info.created_at), allocation_id);
|
|
let p_owner = PriorityOwner::Single(allocation_id);
|
|
let old_kind = info.kind;
|
|
info.kind = kind;
|
|
info.priority = priority;
|
|
let new_key = (info.priority, Reverse(info.created_at), allocation_id);
|
|
// Skip preemption-index updates for unlimited providers: they
|
|
// are intentionally absent from `by_provider` / `priority_index`
|
|
// / `soft_priority_index`, so the old entry was never inserted.
|
|
// Performing the upsert here would re-introduce them into the
|
|
// preemption index and contradict `max_connections: 0`.
|
|
if !info.allocation.is_unlimited_provider() {
|
|
index_updates.push((provider_name, old_key, old_kind, new_key, p_owner, info.kind));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (provider_name, old_key, old_kind, new_key, p_owner, new_kind) in index_updates {
|
|
Self::remove_priority_entry(&mut connections, &provider_name, &old_key, old_kind);
|
|
Self::upsert_priority_entry(&mut connections, &provider_name, new_key, p_owner, new_kind);
|
|
}
|
|
true
|
|
}
|
|
|
|
pub fn reclassify_shared_connection(
|
|
&self,
|
|
subscriber_id: SharedSubscriberId,
|
|
kind: ConnectionKind,
|
|
priority: i8,
|
|
) -> bool {
|
|
let _transition = self.lock_capacity_transition();
|
|
let mut connections = self.write_connections();
|
|
let shared_key = connections.shared.key_by_subscriber.get(&subscriber_id).cloned();
|
|
let Some(shared_key) = shared_key else {
|
|
return false;
|
|
};
|
|
let Some(shared_allocation) = connections.shared.by_key.get_mut(&shared_key) else {
|
|
return false;
|
|
};
|
|
let shared_is_unlimited = shared_allocation.allocation.is_unlimited_provider();
|
|
|
|
let old_priority = shared_allocation.priority;
|
|
let old_kind = shared_allocation.kind;
|
|
let Some(subscriber) = shared_allocation.connections.get_mut(&subscriber_id) else {
|
|
return false;
|
|
};
|
|
subscriber.kind = kind;
|
|
subscriber.priority = priority;
|
|
shared_allocation.kind = Self::shared_effective_kind(&shared_allocation.connections);
|
|
if let Some(new_priority) =
|
|
Self::shared_effective_priority(&shared_allocation.connections, shared_allocation.kind)
|
|
{
|
|
shared_allocation.priority = new_priority;
|
|
}
|
|
let new_priority = shared_allocation.priority;
|
|
let new_kind = shared_allocation.kind;
|
|
let allocation_id = shared_allocation.allocation_id;
|
|
let created_at = shared_allocation.created_at;
|
|
let provider_name = shared_allocation.allocation.get_provider_name();
|
|
let _ = shared_allocation;
|
|
|
|
// Mirror the single-connection rule: skip priority-index rebuild for
|
|
// shared allocations backed by an unlimited provider. The kind/priority
|
|
// transition on the per-subscriber state is still applied above.
|
|
if !shared_is_unlimited {
|
|
if let Some(provider_name) = provider_name {
|
|
let owner = PriorityOwner::Shared(allocation_id);
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
&provider_name,
|
|
&(old_priority, Reverse(created_at), allocation_id),
|
|
old_kind,
|
|
);
|
|
Self::upsert_priority_entry(
|
|
&mut connections,
|
|
&provider_name,
|
|
(new_priority, Reverse(created_at), allocation_id),
|
|
owner,
|
|
new_kind,
|
|
);
|
|
}
|
|
}
|
|
|
|
true
|
|
}
|
|
|
|
/// Promotes exactly the allocation owned by this handle into a shared origin.
|
|
pub fn make_shared_connection(
|
|
&self,
|
|
handle: &ProviderHandle,
|
|
key: &str,
|
|
subscriber_id: SharedSubscriberId,
|
|
) -> bool {
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return false;
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return false;
|
|
}
|
|
let mut connections = self.write_connections();
|
|
if connections.shared.by_key.contains_key(key) {
|
|
return false;
|
|
}
|
|
let Some(info) = connections.single.remove(&handle.allocation_id) else {
|
|
return false;
|
|
};
|
|
if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) {
|
|
set.remove(&handle.allocation_id);
|
|
if set.is_empty() {
|
|
connections.single_by_addr.remove(&info.client_addr);
|
|
}
|
|
}
|
|
let provider_name = info.allocation.get_provider_name().unwrap_or_default();
|
|
if !info.allocation.is_unlimited_provider() {
|
|
if let Some(list) = connections.by_provider.get_mut(&provider_name) {
|
|
list.remove(&handle.allocation_id);
|
|
}
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
&provider_name,
|
|
&(info.priority, Reverse(info.created_at), handle.allocation_id),
|
|
info.kind,
|
|
);
|
|
Self::upsert_priority_entry(
|
|
&mut connections,
|
|
&provider_name,
|
|
(info.priority, Reverse(info.created_at), handle.allocation_id),
|
|
PriorityOwner::Shared(handle.allocation_id),
|
|
info.kind,
|
|
);
|
|
}
|
|
let shared_key: Arc<str> = Arc::from(key);
|
|
connections.shared.by_key.insert(
|
|
Arc::clone(&shared_key),
|
|
SharedAllocation {
|
|
allocation_id: handle.allocation_id,
|
|
origin_subscriber_id: subscriber_id,
|
|
allocation: info.allocation,
|
|
connections: HashMap::from([(
|
|
subscriber_id,
|
|
SharedSubscriber { priority: info.priority, kind: info.kind, addr: handle.client_id },
|
|
)]),
|
|
priority: info.priority,
|
|
kind: info.kind,
|
|
created_at: info.created_at,
|
|
cancel_token: Some(info.cancel_token),
|
|
session_owner: info.session_owner,
|
|
},
|
|
);
|
|
connections.shared.key_by_subscriber.insert(subscriber_id, Arc::clone(&shared_key));
|
|
connections.shared.shared_by_allocation_id.insert(handle.allocation_id, shared_key);
|
|
true
|
|
}
|
|
|
|
pub fn add_shared_connection(
|
|
&self,
|
|
addr: &SocketAddr,
|
|
subscriber_id: SharedSubscriberId,
|
|
key: &str,
|
|
priority: i8,
|
|
kind: ConnectionKind,
|
|
) -> Result<(), String> {
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return Err("Provider manager is shutting down".to_string());
|
|
}
|
|
let _transition = self.lock_capacity_transition();
|
|
if self.is_shutting_down.load(Ordering::Acquire) {
|
|
return Err("Provider manager is shutting down".to_string());
|
|
}
|
|
let mut connections = self.write_connections();
|
|
|
|
// Extract metadata before taking a second mutable borrow on `connections`.
|
|
let metadata = connections.shared.by_key.get(key).map(|s| {
|
|
(
|
|
s.allocation_id,
|
|
s.allocation.get_provider_name().unwrap_or_default(),
|
|
s.priority,
|
|
s.kind,
|
|
s.created_at,
|
|
s.allocation.is_unlimited_provider(),
|
|
)
|
|
});
|
|
|
|
let Some((alloc_id, provider_name, old_priority, old_kind, created_at, shared_is_unlimited)) = metadata else {
|
|
let err =
|
|
format!("Failed to add shared connection for {addr}: url {} not found", sanitize_sensitive_info(key));
|
|
error!("{err}");
|
|
return Err(err);
|
|
};
|
|
|
|
debug_if_enabled!(
|
|
"Shared connection: added addr {addr} provider={} key={}",
|
|
sanitize_sensitive_info(&provider_name),
|
|
sanitize_sensitive_info(key)
|
|
);
|
|
|
|
let Some(shared_allocation) = connections.shared.by_key.get_mut(key) else {
|
|
let err = format!(
|
|
"Failed to add shared connection for {addr}: url {} disappeared during update",
|
|
sanitize_sensitive_info(key)
|
|
);
|
|
error!("{err}");
|
|
return Err(err);
|
|
};
|
|
|
|
shared_allocation.connections.insert(subscriber_id, SharedSubscriber { priority, kind, addr: *addr });
|
|
let new_kind = Self::shared_effective_kind(&shared_allocation.connections);
|
|
let new_priority = Self::shared_effective_priority(&shared_allocation.connections, new_kind);
|
|
shared_allocation.kind = new_kind;
|
|
if let Some(new_priority) = new_priority {
|
|
shared_allocation.priority = new_priority;
|
|
}
|
|
let updated_kind = shared_allocation.kind;
|
|
let updated_priority = shared_allocation.priority;
|
|
let needs_reindex = (updated_priority, updated_kind) != (old_priority, old_kind);
|
|
let _ = shared_allocation;
|
|
// Skip priority-index rebuild for unlimited providers: they are not
|
|
// subject to preemption and therefore must not be in the index.
|
|
if !shared_is_unlimited && needs_reindex {
|
|
Self::remove_priority_entry(
|
|
&mut connections,
|
|
&provider_name,
|
|
&(old_priority, Reverse(created_at), alloc_id),
|
|
old_kind,
|
|
);
|
|
Self::upsert_priority_entry(
|
|
&mut connections,
|
|
&provider_name,
|
|
(updated_priority, Reverse(created_at), alloc_id),
|
|
PriorityOwner::Shared(alloc_id),
|
|
updated_kind,
|
|
);
|
|
}
|
|
|
|
connections.shared.key_by_subscriber.insert(subscriber_id, Arc::from(key));
|
|
Ok(())
|
|
}
|
|
|
|
pub fn get_provider_connections_count(&self) -> usize { self.providers.active_connection_count() }
|
|
|
|
pub fn provider_capacities_for_input(&self, input_name: &Arc<str>) -> Vec<(Arc<str>, usize, usize)> {
|
|
let mut result = Vec::new();
|
|
for provider_name in self.providers.provider_names_for_input(input_name) {
|
|
if let Some((current, max)) = self.providers.provider_capacity(&provider_name) {
|
|
result.push((provider_name, current, max));
|
|
}
|
|
}
|
|
result
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{ActiveProviderManager, ConnectionKind, PlaybackLeaseRef};
|
|
use crate::{ActiveUserManager, EventManager, SharedStreamManager};
|
|
use arc_swap::{ArcSwap, ArcSwapOption};
|
|
use shared::{
|
|
defaults::{default_probe_user_priority, default_user_priority},
|
|
model::{ConfigPaths, InputFetchMethod, InputType},
|
|
utils::Internable,
|
|
};
|
|
use std::{
|
|
collections::{HashMap, HashSet},
|
|
net::SocketAddr,
|
|
sync::{Arc, Weak},
|
|
time::Duration,
|
|
};
|
|
use tuliprox_core::{
|
|
model::{
|
|
AppConfig, Config, ConfigInput, ConfigInputAlias, MediaToolCapabilities, PlaybackKind,
|
|
PlaybackRequestOutcome, ProviderAllocation, SharedSubscriberId, SourcesConfig,
|
|
},
|
|
utils::FileLockManager,
|
|
};
|
|
|
|
fn build_test_app_config(aliases: Option<Vec<ConfigInputAlias>>, max_connections: u16) -> AppConfig {
|
|
let input = Arc::new(ConfigInput {
|
|
id: 1,
|
|
name: "provider_1".intern(),
|
|
input_type: InputType::Xtream,
|
|
headers: HashMap::default(),
|
|
url: "http://provider-1.example".to_string(),
|
|
username: Some("user1".to_string()),
|
|
password: Some("pass1".to_string()),
|
|
enabled: true,
|
|
priority: 0,
|
|
max_connections,
|
|
method: InputFetchMethod::default(),
|
|
aliases,
|
|
..ConfigInput::default()
|
|
});
|
|
|
|
let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() };
|
|
|
|
AppConfig {
|
|
config: Arc::new(ArcSwap::from_pointee(Config::default())),
|
|
sources: Arc::new(ArcSwap::from_pointee(sources)),
|
|
hdhomerun: Arc::new(ArcSwapOption::default()),
|
|
api_proxy: Arc::new(ArcSwapOption::default()),
|
|
file_locks: Arc::new(FileLockManager::default()),
|
|
paths: Arc::new(ArcSwap::from_pointee(ConfigPaths {
|
|
home_path: String::new(),
|
|
config_path: String::new(),
|
|
storage_path: String::new(),
|
|
config_file_path: String::new(),
|
|
sources_file_path: String::new(),
|
|
mapping_file_path: None,
|
|
mapping_files_used: None,
|
|
template_file_path: None,
|
|
template_files_used: None,
|
|
api_proxy_file_path: String::new(),
|
|
custom_stream_response_path: None,
|
|
})),
|
|
custom_stream_response: Arc::new(ArcSwapOption::default()),
|
|
access_token_secret: [0; 32],
|
|
encrypt_secret: [0; 16],
|
|
media_tools: Arc::new(MediaToolCapabilities::new()),
|
|
}
|
|
}
|
|
|
|
fn create_test_app_config_with_dual_provider_pool() -> AppConfig {
|
|
build_test_app_config(
|
|
Some(vec![ConfigInputAlias {
|
|
id: 2,
|
|
name: "provider_2".intern(),
|
|
url: "http://provider-2.example".to_string(),
|
|
username: Some("user2".to_string()),
|
|
password: Some("pass2".to_string()),
|
|
priority: 1,
|
|
max_connections: 1,
|
|
exp_date: None,
|
|
enabled: true,
|
|
stalker: None,
|
|
}]),
|
|
1,
|
|
)
|
|
}
|
|
|
|
fn create_test_app_config_with_capacity_ordered_pool() -> AppConfig {
|
|
build_test_app_config(
|
|
Some(vec![ConfigInputAlias {
|
|
id: 2,
|
|
name: "provider_2".intern(),
|
|
url: "http://provider-2.example".to_string(),
|
|
username: Some("user2".to_string()),
|
|
password: Some("pass2".to_string()),
|
|
priority: 1,
|
|
max_connections: 1,
|
|
exp_date: None,
|
|
enabled: true,
|
|
stalker: None,
|
|
}]),
|
|
3,
|
|
)
|
|
}
|
|
|
|
fn create_test_app_config_single_provider_pool() -> AppConfig { build_test_app_config(None, 1) }
|
|
|
|
fn create_test_app_config_single_unlimited_provider_pool() -> AppConfig { build_test_app_config(None, 0) }
|
|
|
|
/// Pool where the higher-priority provider (A) and its lower-priority alias (B)
|
|
/// each carry their own capacity, mirroring the reported A=2 / B=3 setup.
|
|
fn create_test_app_config_with_pool(primary_max: u16, alias_max: u16) -> AppConfig {
|
|
build_test_app_config(
|
|
Some(vec![ConfigInputAlias {
|
|
id: 2,
|
|
name: "provider_2".intern(),
|
|
url: "http://provider-2.example".to_string(),
|
|
username: Some("user2".to_string()),
|
|
password: Some("pass2".to_string()),
|
|
priority: 1,
|
|
max_connections: alias_max,
|
|
exp_date: None,
|
|
enabled: true,
|
|
stalker: None,
|
|
}]),
|
|
primary_max,
|
|
)
|
|
}
|
|
|
|
#[test]
|
|
fn shared_stream_manager_backref_is_weak_and_does_not_leak() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager));
|
|
let shared = Arc::new(SharedStreamManager::new(Arc::clone(&provider)));
|
|
provider.set_shared_stream_manager(&shared);
|
|
|
|
// The only strong reference to the shared manager is the local `shared`; the
|
|
// provider holds a Weak backref, so dropping `shared` releases both managers.
|
|
drop(shared);
|
|
assert!(provider.shared_stream_manager.get().and_then(Weak::upgrade).is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn confirmed_playbacks_fill_higher_priority_provider_before_alias() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
|
|
let mut selected = Vec::new();
|
|
for index in 0..5 {
|
|
let owner = format!("playback-{index}");
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 50_000 + index));
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(&owner),
|
|
)
|
|
.expect("pool has capacity for five confirmed playbacks");
|
|
let provider = handle.allocation.get_provider_name().expect("provider name");
|
|
// Only real media activity confirms a playback and lets it reserve capacity.
|
|
manager.confirm_playback_activity(&owner);
|
|
selected.push(provider.to_string());
|
|
}
|
|
|
|
assert_eq!(selected, ["provider_1", "provider_1", "provider_2", "provider_2", "provider_2"]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn concurrent_proxy_requests_fill_exact_pool_capacity() -> Result<(), Box<dyn std::error::Error>> {
|
|
let manager = Arc::new(ActiveProviderManager::new(
|
|
&create_test_app_config_with_pool(2, 3),
|
|
&Arc::new(EventManager::new()),
|
|
));
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 50001));
|
|
let mut tasks = tokio::task::JoinSet::new();
|
|
for attempt in 0..32 {
|
|
let manager = Arc::clone(&manager);
|
|
tasks.spawn(async move {
|
|
manager.acquire_connection_with_grace_for_session(
|
|
&Arc::from("provider_1"),
|
|
&addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some(&format!("playback-{attempt}")),
|
|
)
|
|
});
|
|
}
|
|
let mut handles = Vec::new();
|
|
while let Some(result) = tasks.join_next().await {
|
|
if let Some(handle) = result? {
|
|
handles.push(handle);
|
|
}
|
|
}
|
|
assert_eq!(handles.len(), 5);
|
|
assert_eq!(
|
|
handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_1")).count(),
|
|
2
|
|
);
|
|
assert_eq!(
|
|
handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_2")).count(),
|
|
3
|
|
);
|
|
for handle in handles {
|
|
manager.release_handle(&handle);
|
|
}
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn forced_reopen_and_late_cleanup_preserve_other_proxy_requests() -> Result<(), Box<dyn std::error::Error>> {
|
|
let manager =
|
|
ActiveProviderManager::new(&create_test_app_config_with_pool(2, 3), &Arc::new(EventManager::new()));
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 50002));
|
|
let input = Arc::from("provider_1");
|
|
let first = manager
|
|
.acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first"))
|
|
.ok_or("first allocation missing")?;
|
|
let other = manager
|
|
.acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("other"))
|
|
.ok_or("other allocation missing")?;
|
|
manager.release_playback_connections("first", &[addr]);
|
|
assert!(first.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled));
|
|
assert!(!other.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled));
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
let replacement = manager
|
|
.acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first"))
|
|
.ok_or("replacement missing")?;
|
|
manager.refresh_adaptive_playback_lease(&input, "first", tuliprox_core::model::PlaybackKind::LiveHls, 15);
|
|
manager.confirm_playback_activity("first");
|
|
manager.finish_playback_request("first", PlaybackRequestOutcome::ProviderFailed, None);
|
|
assert_eq!(manager.provider_lease_usage(&input).active, 1);
|
|
manager.release_handle(&replacement);
|
|
manager.release_handle(&other);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn release_snapshot_tracks_original_allocation_after_addr_reuse() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let events = Arc::new(EventManager::new());
|
|
let provider = ActiveProviderManager::new(&app_cfg, &events);
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 50_022));
|
|
let input = "provider_1".intern();
|
|
let original = provider
|
|
.acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("old"))
|
|
.expect("original allocation");
|
|
let original_snapshot = provider.release_snapshot_for_addr(&addr);
|
|
assert!(!provider.wait_for_snapshot_release(&original_snapshot, Duration::ZERO).await);
|
|
|
|
provider.release_handle(&original);
|
|
let replacement = provider
|
|
.acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("new"))
|
|
.expect("replacement allocation");
|
|
let replacement_snapshot = provider.release_snapshot_for_addr(&addr);
|
|
assert!(provider.wait_for_snapshot_release(&original_snapshot, Duration::ZERO).await);
|
|
assert!(!provider.wait_for_snapshot_release(&replacement_snapshot, Duration::ZERO).await);
|
|
|
|
let geoip = Arc::new(ArcSwapOption::default());
|
|
let users = ActiveUserManager::new(&Config::default(), &geoip, &events);
|
|
users.set_pending_provider_release("user", original_snapshot.clone()).await;
|
|
users.set_pending_provider_release("user", replacement_snapshot.clone()).await;
|
|
assert!(!users.clear_pending_provider_release("user", &original_snapshot).await);
|
|
assert_eq!(users.pending_provider_release("user").await, Some(replacement_snapshot.clone()));
|
|
assert!(users.clear_pending_provider_release("user", &replacement_snapshot).await);
|
|
assert_eq!(users.pending_provider_release("user").await, None);
|
|
provider.release_handle(&replacement);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn release_snapshot_tracks_shared_subscriber_without_waiting_for_other_subscribers() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let events = Arc::new(EventManager::new());
|
|
let provider = ActiveProviderManager::new(&app_cfg, &events);
|
|
let input = "provider_1".intern();
|
|
let first_addr = SocketAddr::from(([127, 0, 0, 1], 50_023));
|
|
let second_addr = SocketAddr::from(([127, 0, 0, 1], 50_024));
|
|
let first = SharedSubscriberId::from_stream_uid(50_023);
|
|
let second = SharedSubscriberId::from_stream_uid(50_024);
|
|
let origin =
|
|
provider.acquire_connection(&input, &first_addr, 0, ConnectionKind::Normal).expect("shared origin");
|
|
let before_promotion = provider.release_snapshot_for_addr(&first_addr);
|
|
assert!(provider.make_shared_connection(&origin, "shared-release", first));
|
|
provider
|
|
.add_shared_connection(&second_addr, second, "shared-release", 0, ConnectionKind::Normal)
|
|
.expect("second subscriber");
|
|
let snapshot = provider.release_snapshot_for_addr(&first_addr);
|
|
assert!(!provider.wait_for_snapshot_release(&before_promotion, Duration::ZERO).await);
|
|
assert!(!provider.wait_for_snapshot_release(&snapshot, Duration::ZERO).await);
|
|
|
|
provider.release_connection(&first_addr);
|
|
assert!(provider.wait_for_snapshot_release(&before_promotion, Duration::ZERO).await);
|
|
assert!(provider.wait_for_snapshot_release(&snapshot, Duration::ZERO).await);
|
|
assert_eq!(provider.get_provider_connections_count(), 1);
|
|
provider.release_connection(&second_addr);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn manifest_retries_without_media_do_not_reserve_capacity() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner = "hls-cache:shared-session";
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 51_000));
|
|
|
|
// Five manifest responses without a single media segment, all on one owner.
|
|
for _ in 0..5 {
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner),
|
|
)
|
|
.expect("manifest start allocates on the preferred provider");
|
|
assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1"));
|
|
manager.refresh_provider_reservation(&input_name, owner, 15);
|
|
manager.release_connection(&addr);
|
|
}
|
|
|
|
// Unconfirmed leases never reserve capacity, so an unrelated client is served
|
|
// by the higher-priority provider even though its own counters read zero.
|
|
let other = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 51_001)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("other-client"),
|
|
)
|
|
.expect("unrelated client must still get the preferred provider");
|
|
assert_eq!(other.allocation.get_provider_name().as_deref(), Some("provider_1"));
|
|
|
|
// The abandoned starts stop holding anything once their startup deadline passes.
|
|
tokio::time::advance(Duration::from_secs(6)).await;
|
|
manager.prune_expired_leases_now();
|
|
assert!(!manager.is_provider_reserved_for_other_session(&input_name, Some("other-client")));
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn identified_reservation_recreates_a_cleared_lease() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner = "hls-cache:restore-session";
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 53_000));
|
|
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner),
|
|
)
|
|
.expect("allocate");
|
|
manager.refresh_identified_provider_reservation(
|
|
&input_name,
|
|
owner,
|
|
tuliprox_core::model::PlaybackKind::Catchup,
|
|
15,
|
|
);
|
|
assert_eq!(manager.provider_lease_usage(&input_name).starting, 1);
|
|
|
|
// Clearing removes the lease entirely; recreating must bring it back rather
|
|
// than silently no-op like a plain adaptive refresh would.
|
|
let binding_tag = manager.binding_tag_for_owner(owner);
|
|
manager.clear_identified_provider_reservation(owner, &input_name, binding_tag);
|
|
assert_eq!(manager.provider_lease_usage(&input_name).total(), 0);
|
|
|
|
manager.refresh_identified_provider_reservation(
|
|
&input_name,
|
|
owner,
|
|
tuliprox_core::model::PlaybackKind::Catchup,
|
|
15,
|
|
);
|
|
assert_eq!(manager.provider_lease_usage(&input_name).starting, 1);
|
|
|
|
manager.release_handle(&handle);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn reservations_do_not_survive_after_counters_reach_zero() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner = "finished-playback";
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 52_000));
|
|
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner),
|
|
)
|
|
.expect("first playback acquires the preferred provider");
|
|
assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1"));
|
|
manager.refresh_provider_reservation(&input_name, owner, 15);
|
|
manager.confirm_playback_activity(owner);
|
|
manager.release_connection(&addr);
|
|
|
|
// Live TS is not reconnect capable: the confirmed lease is dropped on release,
|
|
// so the provider is immediately free again for the next higher-priority client.
|
|
manager.finish_playback_request(owner, PlaybackRequestOutcome::Completed, None);
|
|
let next = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 52_001)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("next-client"),
|
|
)
|
|
.expect("freed capacity must be selectable again");
|
|
assert_eq!(next.allocation.get_provider_name().as_deref(), Some("provider_1"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn two_playbacks_sharing_one_proxy_socket_keep_separate_slots() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
// Both external clients arrive through the same reverse-proxy peer socket.
|
|
let proxy_addr = SocketAddr::from(([172, 18, 0, 9], 53_000));
|
|
|
|
let first = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&proxy_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("device-one"),
|
|
)
|
|
.expect("first device behind the proxy acquires a slot");
|
|
let second = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&proxy_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("device-two"),
|
|
)
|
|
.expect("second device behind the proxy acquires its own slot");
|
|
assert_ne!(first.allocation_id, second.allocation_id);
|
|
manager.confirm_playback_activity("device-one");
|
|
manager.confirm_playback_activity("device-two");
|
|
|
|
// Releasing one playback must free exactly its own allocation and leave the
|
|
// other device's slot on the same transport untouched.
|
|
manager.release_handle(&first);
|
|
manager.finish_playback_request("device-one", PlaybackRequestOutcome::Completed, None);
|
|
let remaining = manager.provider_capacities_for_input(&input_name);
|
|
let primary = remaining.iter().find(|(name, _, _)| name.as_ref() == "provider_1").expect("primary pool entry");
|
|
assert_eq!(primary.1, 1, "exactly one slot must remain in use on the shared socket");
|
|
assert!(manager.read_leases().lease_of_owner("device-one").is_none());
|
|
assert!(manager.read_leases().lease_of_owner("device-two").is_some());
|
|
|
|
manager.release_handle(&second);
|
|
manager.finish_playback_request("device-two", PlaybackRequestOutcome::Completed, None);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unlimited_provider_connection_is_not_in_priority_index() {
|
|
let app_cfg = create_test_app_config_single_unlimited_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let addr_1: SocketAddr = "127.0.0.1:44001".parse().unwrap();
|
|
let addr_2: SocketAddr = "127.0.0.1:44002".parse().unwrap();
|
|
let addr_3: SocketAddr = "127.0.0.1:44003".parse().unwrap();
|
|
|
|
manager
|
|
.acquire_connection(&input_name, &addr_1, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("acquire #1 on unlimited provider");
|
|
manager
|
|
.acquire_connection(&input_name, &addr_2, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("acquire #2 on unlimited provider");
|
|
manager
|
|
.acquire_connection(&input_name, &addr_3, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("acquire #3 on unlimited provider");
|
|
|
|
{
|
|
let connections = manager.read_connections();
|
|
let priority_tree = connections.priority_index.get(&input_name);
|
|
assert!(
|
|
priority_tree.is_none_or(std::collections::BTreeMap::is_empty),
|
|
"unlimited provider must not be present in priority_index, found {priority_tree:?}"
|
|
);
|
|
let soft_tree = connections.soft_priority_index.get(&input_name);
|
|
assert!(
|
|
soft_tree.is_none_or(std::collections::BTreeMap::is_empty),
|
|
"unlimited provider must not be present in soft_priority_index, found {soft_tree:?}"
|
|
);
|
|
let by_provider = connections.by_provider.get(&input_name);
|
|
assert!(
|
|
by_provider.is_none_or(std::collections::HashSet::is_empty),
|
|
"unlimited provider must not be present in by_provider, found {by_provider:?}"
|
|
);
|
|
}
|
|
|
|
manager.release_connection(&addr_1);
|
|
manager.release_connection(&addr_2);
|
|
manager.release_connection(&addr_3);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn preemption_does_not_select_unlimited_provider_as_victim() {
|
|
let app_cfg = create_test_app_config_single_unlimited_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let low_addr: SocketAddr = "127.0.0.1:45001".parse().unwrap();
|
|
let high_addr: SocketAddr = "127.0.0.1:45002".parse().unwrap();
|
|
|
|
// Acquire one low-priority and one high-priority connection on the unlimited provider.
|
|
let low = manager
|
|
.acquire_connection(&input_name, &low_addr, 50, ConnectionKind::Normal)
|
|
.expect("low-priority acquire on unlimited provider");
|
|
let low_token = low.cancel_token.clone().expect("cancel token for low-priority connection");
|
|
let _high = manager
|
|
.acquire_connection(&input_name, &high_addr, 0, ConnectionKind::Normal)
|
|
.expect("high-priority acquire on unlimited provider");
|
|
|
|
// A request at an even higher priority must not be able to preempt the unlimited
|
|
// provider's connection, because the connection is intentionally absent from the
|
|
// preemption indices.
|
|
let candidate = {
|
|
let connections = manager.read_connections();
|
|
manager.select_preemption_candidate(&connections, &input_name, 0, ConnectionKind::Normal, &HashSet::new())
|
|
};
|
|
assert!(
|
|
candidate.is_none(),
|
|
"select_preemption_candidate must not return an unlimited-provider connection as a victim, got {candidate:?}"
|
|
);
|
|
|
|
// The low-priority connection is still alive and its cancel token has not been fired.
|
|
assert!(
|
|
!low_token.is_cancelled(),
|
|
"low-priority unlimited-provider connection must not be cancelled by preemption"
|
|
);
|
|
|
|
manager.release_connection(&low_addr);
|
|
manager.release_connection(&high_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_force_exact_acquire_does_not_overallocate_busy_provider() {
|
|
let app_cfg = create_test_app_config_with_dual_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let client_1_addr: SocketAddr = "127.0.0.1:40001".parse().unwrap();
|
|
let client_2_addr: SocketAddr = "127.0.0.1:40002".parse().unwrap();
|
|
|
|
let first_alloc = manager
|
|
.acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("client1 initial allocation");
|
|
let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected");
|
|
assert_eq!(pinned_provider.as_ref(), "provider_1");
|
|
|
|
// provider_1 has max_connections=1 and is already in use by client1
|
|
let forced = manager.force_exact_acquire_connection(
|
|
&pinned_provider,
|
|
&client_2_addr,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
);
|
|
assert!(forced.is_none(), "forced exact acquire must not over-allocate busy provider");
|
|
|
|
manager.release_connection(&client_1_addr);
|
|
manager.release_connection(&client_2_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unstarted_series_retry_uses_free_alias_while_started_playback_stays_pinned() {
|
|
let app_cfg = create_test_app_config_with_dual_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input = "provider_1".intern();
|
|
let owner = "series-playback";
|
|
let first_addr: SocketAddr = "127.0.0.1:41010".parse().unwrap();
|
|
let busy_addr: SocketAddr = "127.0.0.1:41011".parse().unwrap();
|
|
let retry_addr: SocketAddr = "127.0.0.1:41012".parse().unwrap();
|
|
|
|
let failed_start = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input,
|
|
&first_addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(owner, PlaybackKind::Series)),
|
|
)
|
|
.expect("first series allocation");
|
|
assert!(manager.should_reuse_playback_provider(owner, &input));
|
|
manager.release_handle(&failed_start);
|
|
assert!(!manager.should_reuse_playback_provider(owner, &input));
|
|
|
|
let busy = manager
|
|
.acquire_connection(&input, &busy_addr, 0, ConnectionKind::Normal)
|
|
.expect("other playback occupies first provider");
|
|
let retry = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input,
|
|
&retry_addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(owner, PlaybackKind::Series)),
|
|
)
|
|
.expect("retry should use free alias");
|
|
let alias = retry.allocation.get_provider_name().expect("alias name");
|
|
assert_eq!(alias.as_ref(), "provider_2");
|
|
assert!(manager.should_reuse_playback_provider(owner, &alias));
|
|
manager.refresh_adaptive_playback_lease(&alias, owner, PlaybackKind::Series, 15);
|
|
manager.confirm_identified_playback_activity(owner, retry.playback_request_id.expect("request id"));
|
|
manager.release_handle(&retry);
|
|
assert!(manager.should_reuse_playback_provider(owner, &alias));
|
|
manager.release_handle(&busy);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_force_session_fallback_uses_different_provider_when_current_is_busy() {
|
|
let app_cfg = create_test_app_config_with_dual_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let client_1_addr: SocketAddr = "127.0.0.1:41001".parse().unwrap();
|
|
let client_2_addr: SocketAddr = "127.0.0.1:41002".parse().unwrap();
|
|
|
|
// Step 1: Client1 starts movie -> provider_1
|
|
let first_alloc = manager
|
|
.acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("client1 initial allocation");
|
|
assert_eq!(first_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
// Step 2: Client1 stops -> release provider_1
|
|
manager.release_connection(&client_1_addr);
|
|
|
|
// Step 3: Client2 starts live -> provider_1
|
|
let live_alloc = manager
|
|
.acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("client2 live allocation");
|
|
let busy_provider = live_alloc.allocation.get_provider_name().expect("provider name expected");
|
|
assert_eq!(busy_provider.as_ref(), input_name.as_ref());
|
|
assert!(manager.is_exhausted(&busy_provider));
|
|
|
|
// Step 4: Client1 restarts same movie.
|
|
// This emulates force-session fallback path by acquiring without provider grace.
|
|
let fallback_alloc = manager
|
|
.acquire_connection_with_grace(&input_name, &client_1_addr, false, 0, ConnectionKind::Normal)
|
|
.expect("client1 fallback allocation without grace");
|
|
let fallback_provider = fallback_alloc.allocation.get_provider_name().expect("fallback provider expected");
|
|
|
|
assert_ne!(fallback_provider.as_ref(), busy_provider.as_ref());
|
|
assert_eq!(fallback_provider.as_ref(), "provider_2");
|
|
|
|
manager.release_connection(&client_1_addr);
|
|
manager.release_connection(&client_2_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_seek_reacquire_stays_on_same_provider_account_until_stop() {
|
|
let app_cfg = create_test_app_config_with_dual_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let client_1_addr: SocketAddr = "127.0.0.1:42001".parse().unwrap();
|
|
let client_2_addr: SocketAddr = "127.0.0.1:42002".parse().unwrap();
|
|
|
|
// Initial playback for client1.
|
|
let first_alloc = manager
|
|
.acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("client1 initial allocation");
|
|
let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected");
|
|
assert_eq!(pinned_provider.as_ref(), "provider_1");
|
|
|
|
// Another client occupies the alternate account while client1 keeps seeking.
|
|
let second_alloc = manager
|
|
.acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("client2 allocation");
|
|
let second_provider = second_alloc.allocation.get_provider_name().expect("provider name expected");
|
|
assert_eq!(second_provider.as_ref(), "provider_2");
|
|
|
|
// Simulate repeated seek/range reconnects for client1:
|
|
// release old connection for the same client, then force exact pinned provider.
|
|
for _ in 0..3 {
|
|
manager.release_connection(&client_1_addr);
|
|
let seek_alloc = manager
|
|
.force_exact_acquire_connection(
|
|
&pinned_provider,
|
|
&client_1_addr,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
)
|
|
.expect("seek reacquire should stay on pinned provider");
|
|
let seek_provider = seek_alloc.allocation.get_provider_name().expect("provider name expected");
|
|
assert_eq!(seek_provider.as_ref(), pinned_provider.as_ref());
|
|
}
|
|
|
|
// Stream stop / cleanup.
|
|
manager.release_connection(&client_1_addr);
|
|
manager.release_connection(&client_2_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_probe_preemption_releases_capacity_and_cancels_immediately() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let user_addr: SocketAddr = "127.0.0.1:43001".parse().unwrap();
|
|
|
|
let probe_handle = manager
|
|
.acquire_connection_for_probe(&input_name, default_probe_user_priority())
|
|
.expect("probe allocation should succeed");
|
|
let probe_token = probe_handle.cancel_token.clone().expect("probe handle must carry cancel token");
|
|
|
|
// User request should preempt probe and immediately acquire released capacity.
|
|
let user_alloc = manager
|
|
.acquire_connection_with_grace(
|
|
&input_name,
|
|
&user_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
)
|
|
.expect("user allocation should preempt probe");
|
|
assert_eq!(user_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
// Cancellation happens inline during preemption; yield once to observe any deferred work.
|
|
tokio::task::yield_now().await;
|
|
assert!(probe_token.is_cancelled(), "probe token should be cancelled immediately after preemption");
|
|
|
|
manager.release_connection(&user_addr);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn test_session_provider_reservation_blocks_other_sessions_until_ttl_expires() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let owner_1 = "session-owner-1";
|
|
let owner_2 = "session-owner-2";
|
|
let addr_1: SocketAddr = "127.0.0.1:43101".parse().unwrap();
|
|
let addr_2: SocketAddr = "127.0.0.1:43102".parse().unwrap();
|
|
|
|
manager.refresh_provider_reservation(&input_name, owner_1, 15);
|
|
manager.confirm_playback_activity(owner_1);
|
|
|
|
let first = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_1,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_1),
|
|
)
|
|
.expect("reserved owner should reacquire its provider");
|
|
assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
manager.release_connection(&addr_1);
|
|
|
|
let blocked = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_2,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_2),
|
|
);
|
|
assert!(blocked.is_none(), "other sessions must not take a reserved provider before TTL expiry");
|
|
|
|
tokio::time::advance(Duration::from_secs(16)).await;
|
|
|
|
let second = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_2,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_2),
|
|
)
|
|
.expect("reservation should expire after TTL");
|
|
assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
manager.release_connection(&addr_2);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn denied_confirmation_followed_by_refresh_cannot_reserve_foreign_slot() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner_x = "session-owner-x";
|
|
let owner_y = "session-owner-y";
|
|
let owner_z = "session-owner-z";
|
|
let addr_y: SocketAddr = "127.0.0.1:43310".parse().unwrap();
|
|
let addr_z: SocketAddr = "127.0.0.1:43311".parse().unwrap();
|
|
|
|
// Y holds the only provider slot.
|
|
let y = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_y,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_y),
|
|
)
|
|
.expect("y acquires the single slot");
|
|
|
|
// X starts a lease and confirms media while Y holds the slot, so its
|
|
// reservation right is denied. A later refresh must not restore it.
|
|
manager.refresh_provider_reservation(&input_name, owner_x, 15);
|
|
manager.confirm_playback_activity(owner_x);
|
|
manager.refresh_provider_reservation(&input_name, owner_x, 15);
|
|
|
|
manager.release_handle(&y);
|
|
|
|
// If X's refresh had re-granted the denied reservation, Z would be blocked here.
|
|
let z = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_z,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_z),
|
|
);
|
|
assert!(z.is_some(), "a denied reservation must not be restored by a later refresh");
|
|
if let Some(z) = z {
|
|
manager.release_handle(&z);
|
|
}
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn untagged_clear_cannot_delete_successor_reservation() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner = "session-owner";
|
|
let addr: SocketAddr = "127.0.0.1:43320".parse().unwrap();
|
|
|
|
// First incarnation acquires, then is removed entirely.
|
|
let first = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner),
|
|
)
|
|
.expect("first acquire");
|
|
manager.release_handle(&first);
|
|
manager.clear_provider_reservation(owner);
|
|
|
|
// Second incarnation acquires a fresh lease on the same account.
|
|
let second = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner),
|
|
)
|
|
.expect("second acquire");
|
|
|
|
// An untagged clear has no delete right and must not remove the successor.
|
|
manager.clear_identified_provider_reservation(owner, &input_name, None);
|
|
assert!(
|
|
manager.binding_tag_for_owner(owner).is_some(),
|
|
"an untagged clear must not delete the successor reservation"
|
|
);
|
|
|
|
manager.release_handle(&second);
|
|
manager.clear_provider_reservation(owner);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn session_reservations_preserve_capacity_and_priority_order() {
|
|
let app_cfg = create_test_app_config_with_capacity_ordered_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
|
|
for index in 0..3 {
|
|
let owner = format!("session-owner-{index}");
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 43200 + index));
|
|
let allocation = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(&owner),
|
|
)
|
|
.expect("preferred provider should retain free capacity for another session");
|
|
assert_eq!(allocation.allocation.get_provider_name().as_deref(), Some("provider_1"));
|
|
manager.refresh_provider_reservation(&input_name, &owner, 15);
|
|
manager.confirm_playback_activity(&owner);
|
|
}
|
|
|
|
let fallback_addr = SocketAddr::from(([127, 0, 0, 1], 43203));
|
|
let fallback = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&fallback_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("session-owner-3"),
|
|
)
|
|
.expect("lower-priority provider should be used after preferred capacity is exhausted");
|
|
assert_eq!(fallback.allocation.get_provider_name().as_deref(), Some("provider_2"));
|
|
|
|
manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 43200)));
|
|
manager.release_connection(&fallback_addr);
|
|
|
|
let foreign_after_release = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&fallback_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("session-owner-4"),
|
|
)
|
|
.expect("idle reservation should protect one preferred-provider slot");
|
|
assert_eq!(foreign_after_release.allocation.get_provider_name().as_deref(), Some("provider_2"));
|
|
|
|
let reserved_owner_addr = SocketAddr::from(([127, 0, 0, 1], 43204));
|
|
let reserved_owner = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&reserved_owner_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("session-owner-0"),
|
|
)
|
|
.expect("reservation owner should reclaim its preferred-provider slot");
|
|
assert_eq!(reserved_owner.allocation.get_provider_name().as_deref(), Some("provider_1"));
|
|
|
|
for index in 1..5 {
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 43200 + index));
|
|
manager.release_connection(&addr);
|
|
}
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn test_unlimited_provider_reservation_does_not_block_other_sessions() {
|
|
let app_cfg = create_test_app_config_single_unlimited_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let owner_1 = "session-owner-1";
|
|
let owner_2 = "session-owner-2";
|
|
let addr_1: SocketAddr = "127.0.0.1:43121".parse().unwrap();
|
|
let addr_2: SocketAddr = "127.0.0.1:43122".parse().unwrap();
|
|
|
|
manager.refresh_provider_reservation(&input_name, owner_1, 15);
|
|
|
|
let first = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_1,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_1),
|
|
)
|
|
.expect("reserved owner should reacquire its unlimited provider");
|
|
assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
let second = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_2,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_2),
|
|
)
|
|
.expect("other sessions should not be blocked by reservations on unlimited providers");
|
|
assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
manager.release_connection(&addr_1);
|
|
manager.release_connection(&addr_2);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn different_session_cannot_take_idle_reservation_from_same_client_family() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let Ok(second_addr) = "127.0.0.1:43112".parse() else {
|
|
return;
|
|
};
|
|
let owner_channel_1 = "client|ua|user|100";
|
|
let owner_channel_2 = "client|ua|user|200";
|
|
|
|
manager.refresh_provider_reservation(&input_name, owner_channel_1, 15);
|
|
manager.confirm_playback_activity(owner_channel_1);
|
|
|
|
let blocked = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&second_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_channel_2),
|
|
);
|
|
assert!(blocked.is_none(), "a related but distinct playback must not steal an idle reservation");
|
|
|
|
manager.clear_provider_reservation(owner_channel_1);
|
|
let acquired = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&second_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_channel_2),
|
|
);
|
|
assert!(acquired.is_some(), "explicitly clearing the old playback should release its provider");
|
|
manager.release_connection(&second_addr);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn concurrent_same_family_playbacks_keep_independent_provider_reservations() {
|
|
let app_cfg = create_test_app_config_with_dual_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let provider_2 = "provider_2".intern();
|
|
let Ok(first_addr) = "127.0.0.1:43121".parse() else {
|
|
return;
|
|
};
|
|
let Ok(second_addr) = "127.0.0.1:43122".parse() else {
|
|
return;
|
|
};
|
|
let owner_channel_1 = "proxy|player|user|100";
|
|
let owner_channel_2 = "proxy|player|user|200";
|
|
|
|
let first = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&first_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_channel_1),
|
|
);
|
|
assert!(first.is_some(), "first playback should acquire the preferred provider");
|
|
let Some(first) = first else {
|
|
return;
|
|
};
|
|
assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
manager.refresh_provider_reservation(&input_name, owner_channel_1, 15);
|
|
manager.confirm_playback_activity(owner_channel_1);
|
|
manager.release_connection(&first_addr);
|
|
|
|
let second = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&second_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_channel_2),
|
|
);
|
|
assert!(second.is_some(), "parallel playback should acquire the remaining provider");
|
|
let Some(second) = second else {
|
|
return;
|
|
};
|
|
assert_eq!(second.allocation.get_provider_name().as_deref(), Some(provider_2.as_ref()));
|
|
manager.refresh_provider_reservation(&provider_2, owner_channel_2, 15);
|
|
manager.confirm_playback_activity(owner_channel_2);
|
|
|
|
let leases = manager.read_leases();
|
|
assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref()));
|
|
assert_eq!(leases.provider_for_owner(owner_channel_2).as_deref(), Some(provider_2.as_ref()));
|
|
drop(leases);
|
|
|
|
manager.clear_provider_reservation(owner_channel_2);
|
|
let leases = manager.read_leases();
|
|
assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref()));
|
|
assert!(leases.lease_of_owner(owner_channel_2).is_none());
|
|
drop(leases);
|
|
|
|
manager.release_connection(&second_addr);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn test_clear_provider_reservation_releases_family_block() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let owner_1 = "session-owner-1";
|
|
let owner_2 = "session-owner-2";
|
|
let addr_2: SocketAddr = "127.0.0.1:43132".parse().unwrap();
|
|
|
|
manager.refresh_provider_reservation(&input_name, owner_1, 15);
|
|
manager.confirm_playback_activity(owner_1);
|
|
|
|
let blocked = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_2,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_2),
|
|
);
|
|
assert!(blocked.is_none(), "reservation should initially block another session");
|
|
|
|
manager.clear_provider_reservation(owner_1);
|
|
|
|
let acquired = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_2,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(owner_2),
|
|
);
|
|
assert!(acquired.is_some(), "clearing reservation should unblock the provider");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_higher_priority_user_preempts_lower_priority_user() {
|
|
// User with priority 5 (low) is connected; user with priority -1 (high) arrives.
|
|
// The low-priority user should be preempted.
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let low_prio_addr: SocketAddr = "127.0.0.1:44001".parse().unwrap();
|
|
let high_prio_addr: SocketAddr = "127.0.0.1:44002".parse().unwrap();
|
|
|
|
// Low-priority user connects (priority 5 = lower importance)
|
|
let low_alloc = manager
|
|
.acquire_connection(&input_name, &low_prio_addr, 5, ConnectionKind::Normal)
|
|
.expect("low-priority user should get connection");
|
|
assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
// Provider is now exhausted
|
|
assert!(manager.is_exhausted(&input_name));
|
|
|
|
// High-priority user arrives (priority -1 = higher importance), should preempt low-priority user
|
|
let high_alloc = manager
|
|
.acquire_connection_with_grace(&input_name, &high_prio_addr, false, -1, ConnectionKind::Normal)
|
|
.expect("high-priority user should preempt low-priority user and get connection");
|
|
assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
manager.release_connection(&high_prio_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_same_priority_user_does_not_preempt() {
|
|
// Two users with the same priority — new one should NOT preempt the existing one.
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let user_1_addr: SocketAddr = "127.0.0.1:45001".parse().unwrap();
|
|
let user_2_addr: SocketAddr = "127.0.0.1:45002".parse().unwrap();
|
|
|
|
// User 1 connects with priority 0
|
|
let alloc1 = manager
|
|
.acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("user1 should get connection");
|
|
assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
// Provider is now exhausted
|
|
assert!(manager.is_exhausted(&input_name));
|
|
|
|
// User 2 arrives with the same priority 0 — should NOT preempt user 1
|
|
let alloc2 = manager.acquire_connection_with_grace(
|
|
&input_name,
|
|
&user_2_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
);
|
|
assert!(alloc2.is_none(), "same-priority user should not preempt existing user");
|
|
|
|
manager.release_connection(&user_1_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_lower_priority_user_does_not_preempt_higher_priority_user() {
|
|
// User with high priority is connected; user with low priority arrives — should NOT preempt.
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let high_prio_addr: SocketAddr = "127.0.0.1:46001".parse().unwrap();
|
|
let low_prio_addr: SocketAddr = "127.0.0.1:46002".parse().unwrap();
|
|
|
|
// High-priority user connects (priority -10)
|
|
let alloc1 = manager
|
|
.acquire_connection(&input_name, &high_prio_addr, -10, ConnectionKind::Normal)
|
|
.expect("high-priority user should get connection");
|
|
assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
// Provider is now exhausted
|
|
assert!(manager.is_exhausted(&input_name));
|
|
|
|
// Low-priority user arrives (priority 10) — should NOT preempt high-priority user
|
|
let alloc2 =
|
|
manager.acquire_connection_with_grace(&input_name, &low_prio_addr, false, 10, ConnectionKind::Normal);
|
|
assert!(alloc2.is_none(), "low-priority user should not preempt high-priority user");
|
|
|
|
manager.release_connection(&high_prio_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_grace_period_triggers_preemption_of_lower_priority() {
|
|
// Provider full, high-prio user arrives with grace allowed,
|
|
// low-prio victim should be evicted and provider should not be over limit.
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let low_prio_addr: SocketAddr = "127.0.0.1:47001".parse().unwrap();
|
|
let high_prio_addr: SocketAddr = "127.0.0.1:47002".parse().unwrap();
|
|
|
|
// Low-priority user connects (priority 20 = low importance)
|
|
let low_alloc = manager
|
|
.acquire_connection(&input_name, &low_prio_addr, 20, ConnectionKind::Normal)
|
|
.expect("low-priority user should get connection");
|
|
assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
let low_token = low_alloc.cancel_token.clone().expect("must have cancel token");
|
|
|
|
// Provider is now exhausted
|
|
assert!(manager.is_exhausted(&input_name));
|
|
|
|
// High-priority user arrives WITH grace allowed (default streaming path)
|
|
// This should get a GracePeriod allocation and then evict the low-prio user
|
|
let high_alloc = manager
|
|
.acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal)
|
|
.expect("high-priority user should get grace allocation and evict low-prio");
|
|
assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
// Low-priority user's cancel token should be cancelled
|
|
assert!(low_token.is_cancelled(), "low-prio user should be cancelled after eviction");
|
|
|
|
// Provider should not be over limit (eviction freed a slot)
|
|
assert!(!manager.is_over_limit(&input_name), "provider should not be over limit after eviction");
|
|
|
|
manager.release_connection(&high_prio_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_equal_priority_user_gets_grace_without_preempting_existing_stream() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let user_1_addr: SocketAddr = "127.0.0.1:48001".parse().unwrap();
|
|
let user_2_addr: SocketAddr = "127.0.0.1:48002".parse().unwrap();
|
|
|
|
// User 1 connects with priority 0
|
|
let alloc1 = manager
|
|
.acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("user1 should get connection");
|
|
assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
let token1 = alloc1.cancel_token.clone().expect("must have cancel token");
|
|
|
|
// Provider is now exhausted
|
|
assert!(manager.is_exhausted(&input_name));
|
|
|
|
// User 2 arrives with the same priority and should be granted grace instead of being rejected.
|
|
let alloc2 = manager
|
|
.acquire_connection(&input_name, &user_2_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("same-priority user should get grace allocation");
|
|
assert!(matches!(alloc2.allocation, ProviderAllocation::GracePeriod(_)));
|
|
assert_eq!(alloc2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
|
|
// User 1 should NOT be cancelled, and the provider should be temporarily over limit.
|
|
assert!(!token1.is_cancelled(), "same-prio user should not be evicted");
|
|
assert!(manager.is_over_limit(&input_name), "provider should be temporarily over limit during grace");
|
|
|
|
manager.release_connection(&user_1_addr);
|
|
manager.release_connection(&user_2_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_higher_priority_user_preempts_first_inserted_low_priority_victim_on_exact_tie() {
|
|
let app_cfg = create_test_app_config_with_dual_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let old_low_addr: SocketAddr = "127.0.0.1:48101".parse().unwrap();
|
|
let new_low_addr: SocketAddr = "127.0.0.1:48102".parse().unwrap();
|
|
let high_prio_addr: SocketAddr = "127.0.0.1:48103".parse().unwrap();
|
|
|
|
// Place two equal-priority low-priority victims on different provider aliases.
|
|
// The test then normalizes their created_at timestamps to an exact tie so the
|
|
// selector must fall back to the final stable tie-break instead of clock order.
|
|
let old_alloc = manager
|
|
.acquire_exact_connection_with_grace(
|
|
&"provider_2".intern(),
|
|
&old_low_addr,
|
|
false,
|
|
20,
|
|
ConnectionKind::Normal,
|
|
)
|
|
.expect("old low-priority allocation should succeed");
|
|
let old_token = old_alloc.cancel_token.clone().expect("old allocation should have cancel token");
|
|
assert_eq!(old_alloc.allocation.get_provider_name().as_deref(), Some("provider_2"));
|
|
|
|
let new_alloc = manager
|
|
.acquire_exact_connection_with_grace(
|
|
&"provider_1".intern(),
|
|
&new_low_addr,
|
|
false,
|
|
20,
|
|
ConnectionKind::Normal,
|
|
)
|
|
.expect("new low-priority allocation should succeed");
|
|
let new_token = new_alloc.cancel_token.clone().expect("new allocation should have cancel token");
|
|
assert_eq!(new_alloc.allocation.get_provider_name().as_deref(), Some("provider_1"));
|
|
|
|
{
|
|
let mut connections = manager.write_connections();
|
|
let old_created_at = connections
|
|
.single
|
|
.get(&old_alloc.allocation_id)
|
|
.map(|info| info.created_at)
|
|
.expect("old allocation should still be registered");
|
|
|
|
let (new_created_at, new_priority) = {
|
|
let info = connections
|
|
.single
|
|
.get_mut(&new_alloc.allocation_id)
|
|
.expect("new allocation should still be registered");
|
|
let original_created_at = info.created_at;
|
|
info.created_at = old_created_at;
|
|
(original_created_at, info.priority)
|
|
};
|
|
|
|
let provider_name = "provider_1".intern();
|
|
let tree =
|
|
connections.priority_index.get_mut(&provider_name).expect("priority index for provider_1 should exist");
|
|
let owner = tree
|
|
.remove(&(new_priority, std::cmp::Reverse(new_created_at), new_alloc.allocation_id))
|
|
.expect("new allocation should still be indexed");
|
|
tree.insert((new_priority, std::cmp::Reverse(old_created_at), new_alloc.allocation_id), owner);
|
|
}
|
|
|
|
assert!(manager.is_exhausted(&input_name));
|
|
|
|
// Higher-priority request should now select the first-inserted victim because
|
|
// priority and created_at are exactly tied across provider aliases.
|
|
let high_alloc = manager
|
|
.acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal)
|
|
.expect("higher-priority request should preempt the first-inserted low-priority victim on exact tie");
|
|
assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some("provider_2"));
|
|
|
|
assert!(old_token.is_cancelled(), "first-inserted low-priority victim should be canceled first on exact tie");
|
|
assert!(!new_token.is_cancelled(), "later allocation should remain active after the stable tie-break");
|
|
|
|
manager.release_connection(&high_prio_addr);
|
|
manager.release_connection(&old_low_addr);
|
|
manager.release_connection(&new_low_addr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn shared_promotion_and_release_preserve_other_allocations_on_same_socket(
|
|
) -> Result<(), Box<dyn std::error::Error>> {
|
|
let app_cfg = build_test_app_config(None, 3);
|
|
let events = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &events);
|
|
let addr = "127.0.0.1:48500".parse()?;
|
|
let input = "provider_1".intern();
|
|
let unrelated = manager
|
|
.acquire_connection(&input, &addr, 0, ConnectionKind::Normal)
|
|
.ok_or("unrelated allocation missing")?;
|
|
let origin =
|
|
manager.acquire_connection(&input, &addr, 5, ConnectionKind::Normal).ok_or("origin allocation missing")?;
|
|
let first = SharedSubscriberId::from_stream_uid(1);
|
|
let second = SharedSubscriberId::from_stream_uid(2);
|
|
let key = "https://example.invalid/shared.ts";
|
|
assert!(manager.make_shared_connection(&origin, key, first));
|
|
assert_eq!(manager.get_provider_connections_count(), 2);
|
|
assert!(manager.read_connections().single.contains_key(&unrelated.allocation_id));
|
|
assert!(manager
|
|
.connections
|
|
.read()
|
|
.unwrap()
|
|
.single_by_addr
|
|
.get(&addr)
|
|
.is_some_and(|allocs| allocs.contains(&unrelated.allocation_id)));
|
|
assert!(!unrelated.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled));
|
|
manager.add_shared_connection(&addr, second, key, 9, ConnectionKind::Soft)?;
|
|
assert!(manager.reclassify_shared_connection(second, ConnectionKind::Normal, 1));
|
|
{
|
|
let connections = manager.read_connections();
|
|
let shared = connections.shared.by_key.get(key).ok_or("shared origin missing")?;
|
|
assert_eq!(shared.connections.get(&first).map(|subscriber| subscriber.priority), Some(5));
|
|
assert_eq!(shared.priority, 1);
|
|
}
|
|
manager.release_shared_connection(second);
|
|
manager.release_shared_connection(second);
|
|
assert_eq!(manager.get_provider_connections_count(), 2);
|
|
assert_eq!(manager.read_connections().shared.by_key.get(key).map(|shared| shared.priority), Some(5));
|
|
manager.release_shared_connection(first);
|
|
manager.release_handle(&origin);
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
manager.release_handle(&unrelated);
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_shared_priority_downgrades_after_high_priority_user_leaves() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let stream_key = "http://example.com/shared/live";
|
|
let addr_a: SocketAddr = "127.0.0.1:48501".parse().unwrap();
|
|
let addr_b: SocketAddr = "127.0.0.1:48502".parse().unwrap();
|
|
|
|
// A starts shared stream with high importance (priority 0).
|
|
let alloc_a = manager
|
|
.acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal)
|
|
.expect("A should get initial connection");
|
|
let shared_token = alloc_a.cancel_token.clone().expect("shared allocation should have cancel token");
|
|
assert!(manager.make_shared_connection(&alloc_a, stream_key, SharedSubscriberId::from_stream_uid(1)));
|
|
|
|
// B joins the same shared stream with lower importance (priority 1).
|
|
let join_result = manager.add_shared_connection(
|
|
&addr_b,
|
|
SharedSubscriberId::from_stream_uid(2),
|
|
stream_key,
|
|
1,
|
|
ConnectionKind::Normal,
|
|
);
|
|
assert!(join_result.is_ok(), "B should join existing shared stream, got: {join_result:?}");
|
|
|
|
// A leaves shared stream. Shared allocation should now inherit B's lower priority.
|
|
manager.release_connection(&addr_a);
|
|
{
|
|
let connections = manager.read_connections();
|
|
let shared = connections.shared.by_key.get(stream_key).expect("shared entry should remain for B");
|
|
assert_eq!(shared.priority, 1, "shared priority must downgrade to remaining subscriber priority");
|
|
}
|
|
|
|
// A starts another stream with higher importance and should preempt B's shared stream.
|
|
let alloc_a2 = manager
|
|
.acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal)
|
|
.expect("A should preempt lower-priority shared stream");
|
|
assert_eq!(alloc_a2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted");
|
|
assert!(!manager.is_over_limit(&input_name), "provider should not remain over limit after preemption");
|
|
|
|
manager.release_connection(&addr_a);
|
|
manager.release_connection(&addr_b);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_higher_priority_user_preempts_shared_stream_with_multiple_subscribers() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let stream_key = "http://example.com/shared/live/multi";
|
|
let addr_a: SocketAddr = "127.0.0.1:48601".parse().unwrap();
|
|
let addr_b: SocketAddr = "127.0.0.1:48602".parse().unwrap();
|
|
let addr_high: SocketAddr = "127.0.0.1:48603".parse().unwrap();
|
|
|
|
let shared_alloc = manager
|
|
.acquire_connection(&input_name, &addr_a, 5, ConnectionKind::Normal)
|
|
.expect("low-priority shared stream should get initial connection");
|
|
let shared_token = shared_alloc.cancel_token.clone().expect("shared allocation should have cancel token");
|
|
assert!(manager.make_shared_connection(&shared_alloc, stream_key, SharedSubscriberId::from_stream_uid(1)));
|
|
|
|
manager
|
|
.add_shared_connection(
|
|
&addr_b,
|
|
SharedSubscriberId::from_stream_uid(2),
|
|
stream_key,
|
|
6,
|
|
ConnectionKind::Normal,
|
|
)
|
|
.expect("second subscriber should join shared stream");
|
|
|
|
let high_alloc = manager
|
|
.acquire_connection(&input_name, &addr_high, 0, ConnectionKind::Normal)
|
|
.expect("higher-priority user should preempt lower-priority shared stream");
|
|
assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref()));
|
|
assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted");
|
|
|
|
{
|
|
let connections = manager.read_connections();
|
|
assert!(
|
|
!connections.shared.by_key.contains_key(stream_key),
|
|
"preempted shared stream must be removed even with multiple subscribers"
|
|
);
|
|
}
|
|
|
|
manager.release_connection(&addr_high);
|
|
manager.release_connection(&addr_a);
|
|
manager.release_connection(&addr_b);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_btree_index_consistent_after_lifecycle() {
|
|
// Verify that the priority_index stays consistent through add, evict, release.
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let addr_a: SocketAddr = "127.0.0.1:49001".parse().unwrap();
|
|
let addr_b: SocketAddr = "127.0.0.1:49002".parse().unwrap();
|
|
|
|
// Add connection A (low priority)
|
|
let alloc_a = manager.acquire_connection(&input_name, &addr_a, 10, ConnectionKind::Normal).expect("alloc_a");
|
|
|
|
// Check index has 1 entry
|
|
{
|
|
let connections = manager.read_connections();
|
|
let tree = connections.priority_index.get(&input_name).expect("index for provider_1");
|
|
assert_eq!(tree.len(), 1, "index should have 1 entry after first allocation");
|
|
}
|
|
|
|
// High-priority user evicts low-priority via grace path
|
|
let alloc_b = manager
|
|
.acquire_connection(&input_name, &addr_b, -5, ConnectionKind::Normal)
|
|
.expect("alloc_b should evict alloc_a");
|
|
|
|
// Check index: should have 1 entry (alloc_a evicted, alloc_b added)
|
|
{
|
|
let connections = manager.read_connections();
|
|
let tree = connections.priority_index.get(&input_name).expect("index for provider_1");
|
|
assert_eq!(tree.len(), 1, "index should have 1 entry after eviction + new allocation");
|
|
// The remaining entry should be alloc_b
|
|
let ((prio, _, _), _) = tree.iter().next().expect("one entry");
|
|
assert_eq!(*prio, -5, "remaining entry should be the high-prio connection");
|
|
}
|
|
|
|
// Release alloc_b
|
|
manager.release_handle(&alloc_b);
|
|
|
|
// Check index: should be empty
|
|
{
|
|
let connections = manager.read_connections();
|
|
let tree = connections.priority_index.get(&input_name);
|
|
let is_empty = tree.is_none_or(std::collections::BTreeMap::is_empty);
|
|
assert!(is_empty, "index should be empty after releasing all connections");
|
|
}
|
|
|
|
// Verify alloc_a handle can be safely released (already evicted - no-op)
|
|
manager.release_handle(&alloc_a);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_preemption_respects_registered_body_owner_completion() {
|
|
let app_cfg = build_test_app_config(None, 1);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let addr_1: SocketAddr = "127.0.0.1:49061".parse().unwrap();
|
|
let addr_2: SocketAddr = "127.0.0.1:49062".parse().unwrap();
|
|
|
|
// 1. First request with prio 5 occupies limit=1
|
|
let handle_1 = manager
|
|
.acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal)
|
|
.expect("first allocation should succeed");
|
|
|
|
// 2. Register body owner on handle_1
|
|
manager.register_body_owner(handle_1.allocation_id);
|
|
|
|
// 3. Higher-priority request with prio -1 arrives without grace while handle_1 is active
|
|
let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal);
|
|
|
|
// 4. Replacement must NOT be admitted before completion
|
|
assert!(handle_2.is_none(), "replacement must not be admitted before victim completion");
|
|
assert!(
|
|
handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(),
|
|
"victim should receive cancel signal"
|
|
);
|
|
assert_eq!(
|
|
handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire),
|
|
tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8,
|
|
"victim close reason must be PriorityPreempted"
|
|
);
|
|
|
|
// 5. Victim completes upstream reading
|
|
handle_1.completion_token.as_ref().expect("completion token").cancel();
|
|
|
|
// Allow background complete_release task to run
|
|
tokio::task::yield_now().await;
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
|
|
// 6. Now replacement request can be admitted
|
|
let handle_2 = manager
|
|
.acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal)
|
|
.expect("replacement should succeed after victim has completed release");
|
|
|
|
manager.release_handle(&handle_2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_preemption_respects_opening_connection_completion() {
|
|
let app_cfg = build_test_app_config(None, 1);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let addr_1: SocketAddr = "127.0.0.1:49063".parse().unwrap();
|
|
let addr_2: SocketAddr = "127.0.0.1:49064".parse().unwrap();
|
|
|
|
// 1. First request with prio 5 occupies limit=1
|
|
let handle_1 = manager
|
|
.acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal)
|
|
.expect("first allocation should succeed");
|
|
manager.mark_opening(handle_1.allocation_id);
|
|
|
|
// Verify lifecycle is Opening and body owner is not yet registered
|
|
{
|
|
let connections = manager.read_connections();
|
|
let info = connections.single.get(&handle_1.allocation_id).expect("handle_1 exists");
|
|
assert_eq!(info.lifecycle, tuliprox_core::model::ConnectionLifecycle::Opening);
|
|
assert!(!info.has_body_owner);
|
|
}
|
|
|
|
// 2. Higher-priority request with prio -1 arrives without grace while handle_1 is in Opening
|
|
let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal);
|
|
|
|
// 3. Replacement must NOT be admitted before completion of opening
|
|
assert!(handle_2.is_none(), "replacement must not be admitted while victim is still opening");
|
|
assert!(
|
|
handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(),
|
|
"victim should receive cancel signal during open"
|
|
);
|
|
assert_eq!(
|
|
handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire),
|
|
tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8,
|
|
"victim close reason must be PriorityPreempted"
|
|
);
|
|
|
|
// 4. Victim's open future completes and signals completion
|
|
handle_1.completion_token.as_ref().expect("completion token").cancel();
|
|
|
|
// Allow background complete_release task to run
|
|
tokio::task::yield_now().await;
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
|
|
// 5. Now replacement request can be admitted
|
|
let handle_2 = manager
|
|
.acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal)
|
|
.expect("replacement should succeed after victim open future has completed release");
|
|
|
|
manager.release_handle(&handle_2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_owner_index_consistent_through_lifecycle() {
|
|
let app_cfg = build_test_app_config(None, 2);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let addr_a: SocketAddr = "127.0.0.1:49051".parse().unwrap();
|
|
let addr_b: SocketAddr = "127.0.0.1:49052".parse().unwrap();
|
|
|
|
let owner_a = "owner-a";
|
|
let owner_b = "owner-b";
|
|
let alloc_a = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_a,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some(owner_a),
|
|
)
|
|
.expect("alloc_a");
|
|
let alloc_b = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr_b,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some(owner_b),
|
|
)
|
|
.expect("alloc_b");
|
|
|
|
{
|
|
let connections = manager.read_connections();
|
|
assert_eq!(connections.by_owner.get(owner_a).map_or(0, HashSet::len), 1, "owner-a has one allocation");
|
|
assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b has one allocation");
|
|
assert!(connections.by_owner.get(owner_a).unwrap().contains(&alloc_a.allocation_id));
|
|
assert!(connections.by_owner.get(owner_b).unwrap().contains(&alloc_b.allocation_id));
|
|
}
|
|
|
|
manager.release_handle(&alloc_a);
|
|
|
|
{
|
|
let connections = manager.read_connections();
|
|
assert!(!connections.by_owner.contains_key(owner_a), "owner-a index is removed after release");
|
|
assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b still indexed");
|
|
}
|
|
|
|
manager.release_handle(&alloc_b);
|
|
{
|
|
let connections = manager.read_connections();
|
|
assert!(connections.by_owner.is_empty(), "owner index is empty after all releases");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_normal_connection_preempts_existing_soft_connection() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let soft_addr: SocketAddr = "127.0.0.1:49011".parse().unwrap();
|
|
let normal_addr: SocketAddr = "127.0.0.1:49012".parse().unwrap();
|
|
|
|
let soft_alloc = manager
|
|
.acquire_connection(&input_name, &soft_addr, default_user_priority(), ConnectionKind::Soft)
|
|
.expect("soft allocation");
|
|
let soft_token = soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token");
|
|
|
|
let normal_alloc = manager
|
|
.acquire_connection(&input_name, &normal_addr, default_user_priority(), ConnectionKind::Normal)
|
|
.expect("normal allocation should preempt soft allocation");
|
|
|
|
tokio::task::yield_now().await;
|
|
assert!(soft_token.is_cancelled(), "soft allocation should be preempted by normal traffic");
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
manager.release_handle(&normal_alloc);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_higher_priority_soft_preempts_lower_priority_soft() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let low_soft_addr: SocketAddr = "127.0.0.1:49013".parse().unwrap();
|
|
let high_soft_addr: SocketAddr = "127.0.0.1:49014".parse().unwrap();
|
|
|
|
let low_soft_alloc = manager
|
|
.acquire_connection(&input_name, &low_soft_addr, 10, ConnectionKind::Soft)
|
|
.expect("low-priority soft allocation");
|
|
let low_soft_token = low_soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token");
|
|
|
|
let high_soft_alloc = manager
|
|
.acquire_connection(&input_name, &high_soft_addr, -5, ConnectionKind::Soft)
|
|
.expect("higher-priority soft allocation should preempt lower-priority soft allocation");
|
|
|
|
tokio::task::yield_now().await;
|
|
assert!(low_soft_token.is_cancelled(), "lower-priority soft allocation should be preempted");
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
manager.release_handle(&high_soft_alloc);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_reclassify_soft_to_normal_prevents_same_priority_preemption() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let promoted_addr: SocketAddr = "127.0.0.1:49015".parse().unwrap();
|
|
let challenger_addr: SocketAddr = "127.0.0.1:49016".parse().unwrap();
|
|
|
|
let promoted_alloc = manager
|
|
.acquire_connection(&input_name, &promoted_addr, default_user_priority(), ConnectionKind::Soft)
|
|
.expect("initial soft allocation");
|
|
let promoted_token = promoted_alloc.cancel_token.clone().expect("soft allocations expose a cancel token");
|
|
|
|
assert!(
|
|
manager.reclassify_connection(&promoted_addr, ConnectionKind::Normal, default_user_priority()),
|
|
"soft allocation should be promotable to normal"
|
|
);
|
|
|
|
let challenger = manager.acquire_connection_with_grace(
|
|
&input_name,
|
|
&challenger_addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
);
|
|
assert!(challenger.is_none(), "same-priority normal traffic should not preempt a promoted normal connection");
|
|
assert!(!promoted_token.is_cancelled(), "promoted connection should remain active");
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
manager.release_handle(&promoted_alloc);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_reclassify_connection_for_owner_preserves_other_owner_on_same_socket() {
|
|
let app_cfg = build_test_app_config(None, 2);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
|
|
let input_name = "provider_1".intern();
|
|
let shared_addr: SocketAddr = "127.0.0.1:49020".parse().unwrap();
|
|
|
|
let alloc_1 = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&shared_addr,
|
|
false,
|
|
-10,
|
|
ConnectionKind::Soft,
|
|
Some(PlaybackLeaseRef::new("owner-1", PlaybackKind::LiveTs)),
|
|
)
|
|
.expect("first soft allocation");
|
|
|
|
let alloc_2 = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&shared_addr,
|
|
false,
|
|
-10,
|
|
ConnectionKind::Soft,
|
|
Some(PlaybackLeaseRef::new("owner-2", PlaybackKind::LiveTs)),
|
|
)
|
|
.expect("second soft allocation");
|
|
|
|
assert!(manager.reclassify_connection_for_owner(&shared_addr, Some("owner-1"), ConnectionKind::Normal, 0));
|
|
|
|
{
|
|
let connections = manager.read_connections();
|
|
let info_1 = connections.single.get(&alloc_1.allocation_id).expect("alloc_1 exists");
|
|
assert_eq!(info_1.kind, ConnectionKind::Normal);
|
|
assert_eq!(info_1.priority, 0);
|
|
|
|
let info_2 = connections.single.get(&alloc_2.allocation_id).expect("alloc_2 exists");
|
|
assert_eq!(info_2.kind, ConnectionKind::Soft);
|
|
assert_eq!(info_2.priority, -10);
|
|
}
|
|
|
|
manager.release_handle(&alloc_1);
|
|
manager.release_handle(&alloc_2);
|
|
}
|
|
|
|
/// A=2 / B=3 pool: five confirmed playbacks exhaust the pool and a sixth start is
|
|
/// rejected; releasing one confirmed playback frees exactly one slot again.
|
|
#[tokio::test]
|
|
async fn sixth_playback_rejected_after_pool_exhausted_by_confirmed_leases() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
|
|
let mut acquired = Vec::new();
|
|
for index in 0..5u16 {
|
|
let owner = format!("playback-{index}");
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 60_000 + index));
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(&owner),
|
|
)
|
|
.expect("pool has capacity for five playbacks");
|
|
manager.confirm_playback_activity(&owner);
|
|
acquired.push((handle, owner));
|
|
}
|
|
|
|
let sixth = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 60_005)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("playback-5"),
|
|
);
|
|
assert!(sixth.is_none(), "sixth start must be rejected once the pool is exhausted");
|
|
|
|
manager.release_handle(&acquired[0].0);
|
|
let replacement = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 60_006)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some("playback-6"),
|
|
);
|
|
assert!(replacement.is_some(), "freed capacity must be selectable again");
|
|
manager.release_handle(&replacement.expect("replacement handle"));
|
|
for (handle, _owner) in &acquired[1..] {
|
|
manager.release_handle(handle);
|
|
}
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn live_hls_lease_owner_preserves_other_playback_identities() {
|
|
let owner = "client|alice|42";
|
|
assert_eq!(super::playback_lease_owner("client|alice|42|hls|0123456789abcdef"), owner);
|
|
for token in [
|
|
owner,
|
|
"client|alice|42|hls|short",
|
|
"client|alice|42|hls|0123456789abcde!",
|
|
"m3u-catchup|client|alice|42|hls|0123456789abcdef",
|
|
"catchup|client|alice|42|hls|0123456789abcdef",
|
|
"hls-cache:client|hls|0123456789abcdef",
|
|
] {
|
|
assert_eq!(super::playback_lease_owner(token), token);
|
|
}
|
|
assert_ne!(
|
|
super::playback_lease_owner("client|alice|42|hls|0123456789abcdef"),
|
|
super::playback_lease_owner("other-client|alice|42|hls|0123456789abcdef")
|
|
);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn live_hls_retries_with_unique_tokens_keep_one_provider_lease() -> Result<(), String> {
|
|
let app_cfg = create_test_app_config_with_pool(2, 4);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input = "provider_1".intern();
|
|
let alias = "provider_2".intern();
|
|
let mut original_tag = None;
|
|
let mut previous_request: Option<(
|
|
String,
|
|
tuliprox_core::model::PlaybackRequestId,
|
|
Option<tuliprox_core::model::ProviderBindingTag>,
|
|
)> = None;
|
|
for attempt in 0..104_u16 {
|
|
let token = format!("client|alice|42|hls|{attempt:016x}");
|
|
let handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input,
|
|
&SocketAddr::from(([172, 18, 0, 9], 55_000 + attempt)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(&token, PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("retry must reuse the reserved provider")?;
|
|
let request_id = handle.playback_request_id.ok_or("identified HLS request")?;
|
|
let provider = handle.allocation.get_provider_name().ok_or("provider")?;
|
|
assert_eq!(provider, input);
|
|
let tag = handle.binding_tag.ok_or("binding tag")?;
|
|
let first_tag = original_tag.get_or_insert(tag);
|
|
assert_eq!(tag.lease_id, first_tag.lease_id);
|
|
manager.refresh_adaptive_playback_lease(&provider, &token, PlaybackKind::LiveHls, 60);
|
|
manager.confirm_identified_playback_activity(&token, request_id);
|
|
assert!(manager.should_reuse_playback_provider(&token, &provider));
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(&token, request_id, PlaybackRequestOutcome::Completed);
|
|
manager.clear_provider_reservation(&token);
|
|
if let Some((old_token, old_request, old_tag)) = previous_request.take() {
|
|
manager.clear_identified_provider_reservation(&old_token, &provider, old_tag);
|
|
manager.finish_identified_playback_request(
|
|
&old_token,
|
|
old_request,
|
|
PlaybackRequestOutcome::ProviderFailed,
|
|
);
|
|
assert!(manager.confirm_identified_playback_activity(&old_token, old_request).is_none());
|
|
}
|
|
previous_request = Some((token, request_id, handle.binding_tag));
|
|
assert_eq!(manager.provider_lease_usage(&input).idle, 1);
|
|
assert_eq!(manager.provider_lease_usage(&alias).total(), 0);
|
|
}
|
|
let foreign = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input,
|
|
&SocketAddr::from(([172, 18, 0, 10], 56_000)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("other-client|alice|42|hls|0123456789abcdef", PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("another playback must retain free provider capacity")?;
|
|
manager.release_handle(&foreign);
|
|
tokio::time::advance(Duration::from_secs(61)).await;
|
|
assert_eq!(manager.provider_lease_usage(&input).total(), 0);
|
|
Ok(())
|
|
}
|
|
|
|
/// Primary `provider_1` with an alias `provider_2` behind it.
|
|
fn alias_pool(primary_max: u16, alias_max: u16) -> (ActiveProviderManager, Arc<str>, Arc<str>) {
|
|
let app_cfg = create_test_app_config_with_pool(primary_max, alias_max);
|
|
let manager = ActiveProviderManager::new(&app_cfg, &Arc::new(EventManager::new()));
|
|
(manager, "provider_1".intern(), "provider_2".intern())
|
|
}
|
|
|
|
fn acquire_live_hls_from_lineup(
|
|
manager: &ActiveProviderManager,
|
|
input: &Arc<str>,
|
|
token: &str,
|
|
port: u16,
|
|
) -> Result<tuliprox_core::model::ProviderHandle, String> {
|
|
manager
|
|
.acquire_connection_with_lease_for_session(
|
|
input,
|
|
&SocketAddr::from(([172, 18, 0, 9], port)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(token, PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or_else(|| format!("no provider for {token}"))
|
|
}
|
|
|
|
/// Starts `token` on the alias while the primary is full, confirms media, ends the
|
|
/// request cleanly, frees the primary and lets the reconnect lease lapse.
|
|
async fn live_hls_playback_on_alias_with_lapsed_lease(
|
|
manager: &ActiveProviderManager,
|
|
input: &Arc<str>,
|
|
alias: &Arc<str>,
|
|
token: &str,
|
|
outcome: PlaybackRequestOutcome,
|
|
) -> Result<(), String> {
|
|
let busy_1 = acquire_live_hls_from_lineup(manager, input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?;
|
|
let busy_2 = acquire_live_hls_from_lineup(manager, input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?;
|
|
assert_eq!(busy_1.allocation.get_provider_name().as_ref(), Some(input));
|
|
assert_eq!(busy_2.allocation.get_provider_name().as_ref(), Some(input));
|
|
|
|
let handle = acquire_live_hls_from_lineup(manager, input, token, 50_010)?;
|
|
let provider = handle.allocation.get_provider_name().ok_or("provider")?;
|
|
assert_eq!(&provider, alias, "full primary must fall back to the alias");
|
|
let request_id = handle.playback_request_id.ok_or("identified request")?;
|
|
manager.refresh_adaptive_playback_lease(&provider, token, PlaybackKind::LiveHls, 15);
|
|
manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?;
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(token, request_id, outcome);
|
|
|
|
manager.release_handle(&busy_1);
|
|
manager.release_handle(&busy_2);
|
|
tokio::time::advance(Duration::from_secs(20)).await;
|
|
assert_eq!(manager.provider_lease_usage(alias).total(), 0, "reconnect lease must have lapsed");
|
|
assert_eq!(manager.provider_lease_usage(input).total(), 0);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn live_hls_reentry_keeps_alias_affinity_after_lease_expiry() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
live_hls_playback_on_alias_with_lapsed_lease(
|
|
&manager,
|
|
&input,
|
|
&alias,
|
|
"client|alice|42|hls|0123456789abcdef",
|
|
PlaybackRequestOutcome::Completed,
|
|
)
|
|
.await?;
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&alias)));
|
|
// Affinity reserves nothing: another playback still gets the free primary slot.
|
|
let other = acquire_live_hls_from_lineup(&manager, &input, "client|dave|42|hls|0123456789abcdef", 50_012)?;
|
|
assert_eq!(other.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
manager.release_handle(&reentry);
|
|
manager.release_handle(&other);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn live_hls_alias_affinity_ends_after_its_window() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
live_hls_playback_on_alias_with_lapsed_lease(
|
|
&manager,
|
|
&input,
|
|
&alias,
|
|
"client|alice|42|hls|0123456789abcdef",
|
|
PlaybackRequestOutcome::Completed,
|
|
)
|
|
.await?;
|
|
tokio::time::advance(Duration::from_secs(shared::defaults::default_provider_affinity_ttl_secs())).await;
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
manager.release_handle(&reentry);
|
|
Ok(())
|
|
}
|
|
|
|
/// Starts `token` on the alias while the primary is full and confirms media. The
|
|
/// request stays attached; the primary is freed again before returning.
|
|
fn confirmed_alias_request(
|
|
manager: &ActiveProviderManager,
|
|
input: &Arc<str>,
|
|
alias: &Arc<str>,
|
|
token: &str,
|
|
) -> Result<tuliprox_core::model::ProviderHandle, String> {
|
|
let busy_1 = acquire_live_hls_from_lineup(manager, input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?;
|
|
let busy_2 = acquire_live_hls_from_lineup(manager, input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?;
|
|
let handle = acquire_live_hls_from_lineup(manager, input, token, 50_010)?;
|
|
assert_eq!(handle.allocation.get_provider_name().as_ref(), Some(alias));
|
|
let request_id = handle.playback_request_id.ok_or("identified request")?;
|
|
manager.refresh_adaptive_playback_lease(alias, token, PlaybackKind::LiveHls, 15);
|
|
manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?;
|
|
manager.release_handle(&busy_1);
|
|
manager.release_handle(&busy_2);
|
|
Ok(handle)
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn provider_failure_after_lease_expiry_ends_alias_affinity() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
let token = "client|alice|42|hls|0123456789abcdef";
|
|
let handle = confirmed_alias_request(&manager, &input, &alias, token)?;
|
|
let request_id = handle.playback_request_id.ok_or("identified request")?;
|
|
manager.release_handle(&handle);
|
|
tokio::time::advance(Duration::from_secs(20)).await;
|
|
manager.prune_expired_leases_now();
|
|
assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired");
|
|
|
|
manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::ProviderFailed);
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
manager.release_handle(&reentry);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn delayed_failure_of_older_binding_keeps_successor_affinity() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
let old_token = "client|alice|42|hls|0123456789abcdef";
|
|
let new_token = "client|alice|42|hls|fedcba9876543210";
|
|
|
|
// An older binding on the primary that never confirmed media.
|
|
let old = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&input,
|
|
&SocketAddr::from(([172, 18, 0, 9], 50_000)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(old_token, PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("old binding")?;
|
|
let old_request = old.playback_request_id.ok_or("old request")?;
|
|
let old_tag = old.binding_tag.ok_or("old tag")?;
|
|
manager.release_handle(&old);
|
|
|
|
let current = confirmed_alias_request(&manager, &input, &alias, new_token)?;
|
|
let current_request = current.playback_request_id.ok_or("current request")?;
|
|
manager.release_handle(¤t);
|
|
manager.finish_identified_playback_request(new_token, current_request, PlaybackRequestOutcome::Completed);
|
|
|
|
// Delayed manifest failures of the older binding arrive after the rebind.
|
|
assert!(!manager.forget_identified_provider_affinity(old_token, &input, old_tag));
|
|
manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::ProviderFailed);
|
|
tokio::time::advance(Duration::from_secs(20)).await;
|
|
assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired");
|
|
manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::ProviderFailed);
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|00000000000000aa", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&alias)));
|
|
manager.release_handle(&reentry);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn administrative_termination_of_hls_token_ends_affinity_and_lease() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
let token = "client|alice|42|hls|0123456789abcdef";
|
|
let handle = confirmed_alias_request(&manager, &input, &alias, token)?;
|
|
let request_id = handle.playback_request_id.ok_or("identified request")?;
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::Completed);
|
|
assert_eq!(manager.provider_lease_usage(&alias).idle, 1);
|
|
|
|
// The bare-owner clear has no delete right for public HLS tokens.
|
|
manager.clear_provider_reservation(token);
|
|
assert_eq!(manager.provider_lease_usage(&alias).idle, 1);
|
|
manager.terminate_identified_playback_owner(token);
|
|
assert_eq!(manager.provider_lease_usage(&alias).total(), 0);
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
manager.release_handle(&reentry);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn stale_hls_token_termination_keeps_newer_binding() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
let old_token = "client|alice|42|hls|0123456789abcdef";
|
|
let new_token = "client|alice|42|hls|fedcba9876543210";
|
|
|
|
let old = confirmed_alias_request(&manager, &input, &alias, old_token)?;
|
|
let old_request = old.playback_request_id.ok_or("old request")?;
|
|
manager.release_handle(&old);
|
|
manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::Completed);
|
|
|
|
// The retry re-enters the idle lease and starts a new binding incarnation.
|
|
let new = acquire_live_hls_from_lineup(&manager, &input, new_token, 50_011)?;
|
|
assert_eq!(new.allocation.get_provider_name(), Some(Arc::clone(&alias)));
|
|
assert_ne!(new.binding_tag, old.binding_tag);
|
|
let new_request = new.playback_request_id.ok_or("new request")?;
|
|
manager.refresh_adaptive_playback_lease(&alias, new_token, PlaybackKind::LiveHls, 15);
|
|
manager.confirm_identified_playback_activity(new_token, new_request).ok_or("confirmation")?;
|
|
manager.release_handle(&new);
|
|
manager.finish_identified_playback_request(new_token, new_request, PlaybackRequestOutcome::Completed);
|
|
assert_eq!(manager.provider_lease_usage(&alias).idle, 1);
|
|
|
|
assert!(!manager.terminate_identified_playback_owner(old_token));
|
|
assert_eq!(manager.provider_lease_usage(&alias).idle, 1, "stale token must keep the newer lease");
|
|
assert_eq!(manager.provider_affinity_for_owner(new_token), Some(Arc::clone(&alias)));
|
|
|
|
assert!(manager.terminate_identified_playback_owner(new_token));
|
|
assert_eq!(manager.provider_lease_usage(&alias).total(), 0);
|
|
assert_eq!(manager.provider_affinity_for_owner(new_token), None);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn terminating_token_of_expired_binding_ends_its_affinity_only() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
let token = "client|alice|42|hls|0123456789abcdef";
|
|
let handle = confirmed_alias_request(&manager, &input, &alias, token)?;
|
|
manager.release_handle(&handle);
|
|
tokio::time::advance(Duration::from_secs(20)).await;
|
|
assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired");
|
|
|
|
assert!(!manager.terminate_identified_playback_owner("client|alice|42|hls|00000000000000aa"));
|
|
assert_eq!(manager.provider_affinity_for_owner(token), Some(Arc::clone(&alias)));
|
|
assert!(manager.terminate_identified_playback_owner(token));
|
|
assert_eq!(manager.provider_affinity_for_owner(token), None);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn finished_retry_tokens_do_not_accumulate_on_an_active_binding() -> Result<(), String> {
|
|
let app_cfg = create_test_app_config_with_pool(2, 0);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input = "provider_1".intern();
|
|
let first = "client|alice|42|hls|0000000000000000";
|
|
// One request stays in flight for the whole run and keeps the binding active.
|
|
let anchor = acquire_live_hls_from_lineup(&manager, &input, first, 50_000)?;
|
|
let anchor_request = anchor.playback_request_id.ok_or("anchor request")?;
|
|
manager.refresh_adaptive_playback_lease(&input, first, PlaybackKind::LiveHls, 15);
|
|
manager.confirm_identified_playback_activity(first, anchor_request).ok_or("confirmation")?;
|
|
|
|
let mut last_token = String::new();
|
|
for attempt in 1..=1_000_u32 {
|
|
last_token = format!("client|alice|42|hls|{attempt:016x}");
|
|
let handle = acquire_live_hls_from_lineup(&manager, &input, &last_token, 50_001)?;
|
|
assert_eq!(handle.binding_tag, anchor.binding_tag, "retries share the active binding");
|
|
let request_id = handle.playback_request_id.ok_or("retry request")?;
|
|
manager.refresh_adaptive_playback_lease(&input, &last_token, PlaybackKind::LiveHls, 15);
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(&last_token, request_id, PlaybackRequestOutcome::Completed);
|
|
tokio::time::advance(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
let claims = manager.write_leases().request_token_claims(super::playback_lease_owner(first));
|
|
assert!(claims <= 17, "only the anchor and recently finished retries may hold a claim, got {claims}");
|
|
// The anchor keeps its claim while its request runs; a recent retry still has its cleanup right.
|
|
assert!(manager.write_leases().request_token_claims(super::playback_lease_owner(first)) >= 2);
|
|
assert!(manager.terminate_identified_playback_owner(&last_token));
|
|
manager.release_handle(&anchor);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn affinity_never_forces_grace_over_allocation_while_primary_is_free() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 1);
|
|
live_hls_playback_on_alias_with_lapsed_lease(
|
|
&manager,
|
|
&input,
|
|
&alias,
|
|
"client|alice|42|hls|0123456789abcdef",
|
|
PlaybackRequestOutcome::Completed,
|
|
)
|
|
.await?;
|
|
let alias_holder = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&alias,
|
|
&SocketAddr::from(([172, 18, 0, 9], 50_020)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("alias holder")?;
|
|
|
|
// Grace is enabled by default and allowed for this request.
|
|
let reentry = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input,
|
|
&SocketAddr::from(([172, 18, 0, 9], 50_011)),
|
|
true,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("client|alice|42|hls|fedcba9876543210", PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("re-entry")?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
assert!(
|
|
matches!(reentry.allocation, ProviderAllocation::Available(_)),
|
|
"a free primary slot beats a grace allocation"
|
|
);
|
|
manager.release_handle(&reentry);
|
|
manager.release_handle(&alias_holder);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn failure_on_fallback_provider_keeps_preference_for_preferred_provider() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 1);
|
|
let token = "client|alice|42|hls|0123456789abcdef";
|
|
live_hls_playback_on_alias_with_lapsed_lease(
|
|
&manager,
|
|
&input,
|
|
&alias,
|
|
token,
|
|
PlaybackRequestOutcome::Completed,
|
|
)
|
|
.await?;
|
|
let alias_holder = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&alias,
|
|
&SocketAddr::from(([172, 18, 0, 9], 50_020)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("alias holder")?;
|
|
|
|
// The full alias sends the re-entry to the primary, which then fails before media.
|
|
let fallback = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(fallback.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
let fallback_request = fallback.playback_request_id.ok_or("fallback request")?;
|
|
manager.release_handle(&fallback);
|
|
manager.finish_identified_playback_request(
|
|
"client|alice|42|hls|fedcba9876543210",
|
|
fallback_request,
|
|
PlaybackRequestOutcome::FailedBeforeMedia,
|
|
);
|
|
|
|
assert_eq!(manager.provider_affinity_for_owner(token), Some(Arc::clone(&alias)));
|
|
manager.release_handle(&alias_holder);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn live_ts_playback_records_no_provider_affinity() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
let busy_1 = acquire_live_hls_from_lineup(&manager, &input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?;
|
|
let busy_2 = acquire_live_hls_from_lineup(&manager, &input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?;
|
|
let owner = "ts-session-token";
|
|
let handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input,
|
|
&SocketAddr::from(([172, 18, 0, 9], 50_010)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveTs)),
|
|
)
|
|
.ok_or("ts playback")?;
|
|
assert_eq!(handle.allocation.get_provider_name(), Some(Arc::clone(&alias)));
|
|
let request_id = handle.playback_request_id.ok_or("ts request")?;
|
|
manager.confirm_identified_playback_activity(owner, request_id).ok_or("confirmation")?;
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ClientClosed);
|
|
manager.release_handle(&busy_1);
|
|
manager.release_handle(&busy_2);
|
|
|
|
assert_eq!(manager.provider_affinity_for_owner(owner), None, "live TS has nothing to return to");
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn huge_reconnect_and_affinity_windows_never_panic() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
manager.write_leases().set_affinity_ttl(crate::provider_leases::ProviderAffinityTtl(u64::MAX));
|
|
let token = "client|alice|42|hls|0123456789abcdef";
|
|
let handle = acquire_live_hls_from_lineup(&manager, &input, token, 50_010)?;
|
|
let provider = handle.allocation.get_provider_name().ok_or("provider")?;
|
|
let request_id = handle.playback_request_id.ok_or("request")?;
|
|
manager.refresh_adaptive_playback_lease(&provider, token, PlaybackKind::LiveHls, u64::MAX);
|
|
manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?;
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::Completed);
|
|
assert_eq!(manager.provider_affinity_for_owner(token), Some(provider));
|
|
assert_eq!(manager.provider_lease_usage(&alias).total(), 0);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn zero_provider_affinity_ttl_ends_affinity_with_reconnect_window() -> Result<(), String> {
|
|
let app_cfg = create_test_app_config_with_pool(2, 0);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
manager.write_leases().set_affinity_ttl(crate::provider_leases::ProviderAffinityTtl(0));
|
|
let input = "provider_1".intern();
|
|
let alias = "provider_2".intern();
|
|
live_hls_playback_on_alias_with_lapsed_lease(
|
|
&manager,
|
|
&input,
|
|
&alias,
|
|
"client|alice|42|hls|0123456789abcdef",
|
|
PlaybackRequestOutcome::Completed,
|
|
)
|
|
.await?;
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
manager.release_handle(&reentry);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn live_hls_provider_failure_ends_alias_affinity() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 0);
|
|
live_hls_playback_on_alias_with_lapsed_lease(
|
|
&manager,
|
|
&input,
|
|
&alias,
|
|
"client|alice|42|hls|0123456789abcdef",
|
|
PlaybackRequestOutcome::ProviderFailed,
|
|
)
|
|
.await?;
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
manager.release_handle(&reentry);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn live_hls_full_affinity_provider_falls_back_to_lineup() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 1);
|
|
live_hls_playback_on_alias_with_lapsed_lease(
|
|
&manager,
|
|
&input,
|
|
&alias,
|
|
"client|alice|42|hls|0123456789abcdef",
|
|
PlaybackRequestOutcome::Completed,
|
|
)
|
|
.await?;
|
|
let alias_holder = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&alias,
|
|
&SocketAddr::from(([172, 18, 0, 9], 50_020)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("alias holder")?;
|
|
|
|
let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?;
|
|
assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input)));
|
|
manager.release_handle(&reentry);
|
|
manager.release_handle(&alias_holder);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn live_hls_socket_cleanup_only_cancels_its_public_session() -> Result<(), String> {
|
|
let app_cfg = create_test_app_config_with_pool(2, 4);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input = "provider_1".intern();
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 55_000));
|
|
let old = "client|alice|42|hls|0123456789abcdef";
|
|
let new = "client|alice|42|hls|fedcba9876543210";
|
|
let first = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&input,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(old, PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("old allocation")?;
|
|
let next = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&input,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(new, PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("new allocation")?;
|
|
assert_eq!(first.binding_tag, next.binding_tag);
|
|
assert_eq!(manager.provider_lease_usage(&input).total(), 1);
|
|
manager.release_playback_connections(old, &[addr]);
|
|
manager.release_playback_connections_await(old, &[addr]).await;
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
assert!(!next.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled));
|
|
manager.release_handle(&next);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn live_hls_old_token_cleanup_cannot_mutate_rebound_retry() -> Result<(), String> {
|
|
let (manager, input, alias) = alias_pool(2, 4);
|
|
let old = "client|alice|42|hls|0123456789abcdef";
|
|
let new = "client|alice|42|hls|fedcba9876543210";
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 55_000));
|
|
let first = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&input,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(old, PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("initial allocation")?;
|
|
let old_request = first.playback_request_id.ok_or("old request")?;
|
|
manager.refresh_adaptive_playback_lease(&input, old, PlaybackKind::LiveHls, 60);
|
|
manager.confirm_identified_playback_activity(old, old_request);
|
|
manager.release_handle(&first);
|
|
let next = manager
|
|
.acquire_exact_connection_with_lease_for_session(
|
|
&alias,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(new, PlaybackKind::LiveHls)),
|
|
)
|
|
.ok_or("rebound allocation")?;
|
|
let new_request = next.playback_request_id.ok_or("new request")?;
|
|
assert_ne!(first.binding_tag, next.binding_tag);
|
|
manager.refresh_adaptive_playback_lease(&alias, new, PlaybackKind::LiveHls, 60);
|
|
manager.confirm_identified_playback_activity(new, new_request);
|
|
manager.release_handle(&next);
|
|
manager.finish_identified_playback_request(new, new_request, PlaybackRequestOutcome::Completed);
|
|
let before = manager.binding_tag_for_owner(new);
|
|
manager.clear_provider_reservation(old);
|
|
manager.clear_identified_provider_reservation(old, &input, first.binding_tag);
|
|
manager.finish_identified_playback_request(old, old_request, PlaybackRequestOutcome::ProviderFailed);
|
|
manager.refresh_playback_lease(
|
|
&input,
|
|
&PlaybackLeaseRef { owner: old, kind: PlaybackKind::LiveHls, request_id: old_request },
|
|
0,
|
|
);
|
|
assert!(manager.confirm_identified_playback_activity(old, old_request).is_none());
|
|
assert_eq!(manager.binding_tag_for_owner(new), before);
|
|
assert_eq!(manager.provider_lease_usage(&input).total(), 0);
|
|
assert_eq!(manager.provider_lease_usage(&alias).idle, 1);
|
|
Ok(())
|
|
}
|
|
|
|
/// Confirmed reconnect-capable lease, physical handle release first, then the real
|
|
/// provider-error outcome: the error must reach the lease and leave no idle reserve.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn provider_error_cleanup_after_physical_release_does_not_keep_idle_lease() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner = "r5-error-owner";
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 55_000));
|
|
let handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)),
|
|
)
|
|
.expect("acquire a reconnect-capable slot");
|
|
let request_id = handle.playback_request_id.expect("identified request id");
|
|
manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15);
|
|
manager.confirm_playback_activity(owner);
|
|
|
|
// Physical release must not conclude the request; the outcome is decided later.
|
|
manager.release_handle(&handle);
|
|
assert_eq!(
|
|
manager.provider_lease_usage(&input_name).active,
|
|
1,
|
|
"physical release must not finish a confirmed lease"
|
|
);
|
|
|
|
manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed);
|
|
assert_eq!(
|
|
manager.provider_lease_usage(&input_name).total(),
|
|
0,
|
|
"provider error must not keep an idle reconnect lease"
|
|
);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn stale_refresh_does_not_recreate_terminal_request() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner = "terminal-refresh-owner";
|
|
let handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 55_002)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)),
|
|
)
|
|
.expect("acquire identified request");
|
|
let request_id = handle.playback_request_id.expect("identified request id");
|
|
let lease_ref = PlaybackLeaseRef { owner, kind: PlaybackKind::LiveHls, request_id };
|
|
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed);
|
|
assert_eq!(manager.provider_lease_usage(&input_name).total(), 0);
|
|
|
|
manager.refresh_playback_lease(&input_name, &lease_ref, 15);
|
|
manager.confirm_identified_playback_activity(owner, request_id);
|
|
assert_eq!(manager.provider_lease_usage(&input_name).total(), 0);
|
|
}
|
|
|
|
/// Counterexample: the same sequence with a clean end keeps the configured idle window.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn clean_cleanup_after_physical_release_keeps_configured_idle_window() {
|
|
let app_cfg = create_test_app_config_with_pool(2, 3);
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
let owner = "r5-clean-owner";
|
|
let addr = SocketAddr::from(([172, 18, 0, 9], 55_001));
|
|
|
|
let handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)),
|
|
)
|
|
.expect("acquire a reconnect-capable slot");
|
|
let request_id = handle.playback_request_id.expect("identified request id");
|
|
manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15);
|
|
manager.confirm_playback_activity(owner);
|
|
|
|
manager.release_handle(&handle);
|
|
manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::Completed);
|
|
let usage = manager.provider_lease_usage(&input_name);
|
|
assert_eq!(usage.active, 0, "clean end must move the lease out of active");
|
|
assert_eq!(usage.idle, 1, "clean end must keep the configured reconnect window");
|
|
}
|
|
|
|
/// max=1: X fetch → physical release → Y acquire → X first byte within the startup
|
|
/// deadline. X's late confirmation records media but must not over-commit Y's slot.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn late_first_byte_cannot_reserve_slot_taken_by_other_playback() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
|
|
let x_handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 40_000)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)),
|
|
)
|
|
.expect("x acquires the only slot");
|
|
manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15);
|
|
manager.release_handle(&x_handle);
|
|
|
|
let y_handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 40_001)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("y", PlaybackKind::LiveHls)),
|
|
)
|
|
.expect("y acquires the now-free slot");
|
|
|
|
manager.confirm_playback_activity("x");
|
|
assert_eq!(manager.get_provider_connections_count(), 1, "y still holds the only slot");
|
|
assert!(
|
|
!manager.is_provider_reserved_for_other_session(&input_name, Some("y")),
|
|
"x's late confirmation must not over-commit the provider"
|
|
);
|
|
|
|
manager.confirm_playback_activity("y");
|
|
manager.release_handle(&y_handle);
|
|
manager.finish_playback_request("y", PlaybackRequestOutcome::Completed, None);
|
|
}
|
|
|
|
/// Counterexample: the same X fetch → release → first-byte sequence without a
|
|
/// competing Y keeps the free slot as a real reservation.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn first_byte_reserves_when_slot_still_free() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = ActiveProviderManager::new(&app_cfg, &event_manager);
|
|
let input_name = "provider_1".intern();
|
|
|
|
let x_handle = manager
|
|
.acquire_connection_with_lease_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([172, 18, 0, 9], 40_002)),
|
|
false,
|
|
default_user_priority(),
|
|
ConnectionKind::Normal,
|
|
Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)),
|
|
)
|
|
.expect("x acquires the only slot");
|
|
manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15);
|
|
manager.release_handle(&x_handle);
|
|
|
|
// No competing playback took the slot, so X's first byte reserves it.
|
|
manager.confirm_playback_activity("x");
|
|
assert!(
|
|
manager.is_provider_reserved_for_other_session(&input_name, Some("other")),
|
|
"a free slot must be reservable by the confirming playback"
|
|
);
|
|
|
|
manager.finish_playback_request("x", PlaybackRequestOutcome::Completed, None);
|
|
}
|
|
|
|
/// The RAII owner releases the allocation synchronously even when dropped outside
|
|
/// a tokio runtime, so a body/context drop can never leak a provider slot.
|
|
#[test]
|
|
fn managed_handle_drop_outside_runtime_releases_allocation() {
|
|
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
|
|
let (manager, managed_handle) = rt.block_on(async {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager));
|
|
let input_name = "provider_1".intern();
|
|
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([127, 0, 0, 1], 45_000)),
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some("managed-owner"),
|
|
)
|
|
.expect("allocation should succeed");
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
let managed_handle = super::ManagedProviderHandle::new(Arc::clone(&manager), handle);
|
|
(manager, managed_handle)
|
|
});
|
|
|
|
drop(managed_handle);
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
}
|
|
|
|
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)]
|
|
fn latency_percentile(sorted: &[u64], p: f64) -> u64 {
|
|
if sorted.is_empty() {
|
|
return 0;
|
|
}
|
|
let index = ((sorted.len() as f64 - 1.0) * p).round() as usize;
|
|
sorted[index.min(sorted.len() - 1)]
|
|
}
|
|
|
|
#[cfg(target_os = "linux")]
|
|
fn resident_set_kib() -> Option<u64> {
|
|
let Ok(content) = std::fs::read_to_string("/proc/self/status") else {
|
|
return None;
|
|
};
|
|
for line in content.lines() {
|
|
if let Some(rest) = line.strip_prefix("VmRSS:") {
|
|
return rest.split_whitespace().next().and_then(|v| v.parse::<u64>().ok());
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
#[cfg(not(target_os = "linux"))]
|
|
fn resident_set_kib() -> Option<u64> { None }
|
|
|
|
struct LeaseWorkloadStats {
|
|
ops: u64,
|
|
elapsed_secs: f64,
|
|
acquire_p50: u64,
|
|
acquire_p95: u64,
|
|
acquire_p99: u64,
|
|
confirm_p50: u64,
|
|
confirm_p95: u64,
|
|
confirm_p99: u64,
|
|
release_p50: u64,
|
|
release_p95: u64,
|
|
release_p99: u64,
|
|
}
|
|
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
async fn measure_lease_workload(
|
|
manager: &Arc<ActiveProviderManager>,
|
|
input_name: &Arc<str>,
|
|
concurrency: usize,
|
|
rounds_per_task: usize,
|
|
) -> LeaseWorkloadStats {
|
|
let samples = Arc::new(std::sync::Mutex::new(Vec::<(u64, u64, u64)>::new()));
|
|
let started = std::time::Instant::now();
|
|
let mut tasks = tokio::task::JoinSet::new();
|
|
for task_index in 0..concurrency {
|
|
let manager = Arc::clone(manager);
|
|
let input_name = Arc::clone(input_name);
|
|
let samples = Arc::clone(&samples);
|
|
tasks.spawn(async move {
|
|
for round in 0..rounds_per_task {
|
|
let owner = format!("bench-{task_index}-{round}");
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + (task_index as u16)));
|
|
let acquire_at = std::time::Instant::now();
|
|
let Some(handle) = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some(&owner),
|
|
) else {
|
|
continue;
|
|
};
|
|
let acquire_us = acquire_at.elapsed().as_micros() as u64;
|
|
|
|
let confirm_at = std::time::Instant::now();
|
|
manager.confirm_playback_activity(&owner);
|
|
let confirm_us = confirm_at.elapsed().as_micros() as u64;
|
|
|
|
let release_at = std::time::Instant::now();
|
|
manager.release_handle(&handle);
|
|
let release_us = release_at.elapsed().as_micros() as u64;
|
|
|
|
// A physical release never concludes the request; model the real
|
|
// lifecycle by finishing with a clean outcome so the lease returns
|
|
// to baseline (LiveTs is not reconnect-capable, so it is removed).
|
|
manager.finish_playback_request(&owner, PlaybackRequestOutcome::Completed, None);
|
|
|
|
samples.lock().unwrap().push((acquire_us, confirm_us, release_us));
|
|
}
|
|
});
|
|
}
|
|
while let Some(result) = tasks.join_next().await {
|
|
result.expect("benchmark task must not fail");
|
|
}
|
|
|
|
let elapsed_secs = started.elapsed().as_secs_f64();
|
|
let samples = Arc::try_unwrap(samples).expect("samples unique").into_inner().unwrap();
|
|
let mut acquire = Vec::with_capacity(samples.len());
|
|
let mut confirm = Vec::with_capacity(samples.len());
|
|
let mut release = Vec::with_capacity(samples.len());
|
|
for (a, c, r) in samples {
|
|
acquire.push(a);
|
|
confirm.push(c);
|
|
release.push(r);
|
|
}
|
|
acquire.sort_unstable();
|
|
confirm.sort_unstable();
|
|
release.sort_unstable();
|
|
let ops = acquire.len() as u64;
|
|
LeaseWorkloadStats {
|
|
ops,
|
|
elapsed_secs,
|
|
acquire_p50: latency_percentile(&acquire, 0.50),
|
|
acquire_p95: latency_percentile(&acquire, 0.95),
|
|
acquire_p99: latency_percentile(&acquire, 0.99),
|
|
confirm_p50: latency_percentile(&confirm, 0.50),
|
|
confirm_p95: latency_percentile(&confirm, 0.95),
|
|
confirm_p99: latency_percentile(&confirm, 0.99),
|
|
release_p50: latency_percentile(&release, 0.50),
|
|
release_p95: latency_percentile(&release, 0.95),
|
|
release_p99: latency_percentile(&release, 0.99),
|
|
}
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "manager microbenchmark: cargo +stable test -p tuliprox-session --release -- --ignored --nocapture bench_provider_lease_microbenchmark"]
|
|
#[allow(clippy::cast_precision_loss)]
|
|
async fn bench_provider_lease_microbenchmark() {
|
|
let manager = Arc::new(ActiveProviderManager::new(
|
|
&create_test_app_config_single_unlimited_provider_pool(),
|
|
&Arc::new(EventManager::new()),
|
|
));
|
|
let input_name: Arc<str> = "provider_1".intern();
|
|
|
|
// Warm-up before measurement so allocator and lock caches are exercised.
|
|
for index in 0..64u16 {
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + index));
|
|
let Some(handle) = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
None,
|
|
) else {
|
|
continue;
|
|
};
|
|
manager.release_handle(&handle);
|
|
}
|
|
|
|
let baseline_rss_kib = resident_set_kib();
|
|
eprintln!("provider lease manager microbenchmark (unlimited provider, in-process, no HTTP bodies)");
|
|
for concurrency in [1usize, 5, 50, 200] {
|
|
let stats = measure_lease_workload(&manager, &input_name, concurrency, 10).await;
|
|
assert_eq!(
|
|
stats.ops,
|
|
(concurrency * 10) as u64,
|
|
"unexpected acquire failures at concurrency {concurrency}"
|
|
);
|
|
let throughput = stats.ops as f64 / stats.elapsed_secs.max(f64::EPSILON);
|
|
let rss_delta_kib = resident_set_kib()
|
|
.zip(baseline_rss_kib)
|
|
.map_or_else(|| "unsupported".to_string(), |(rss, baseline)| rss.saturating_sub(baseline).to_string());
|
|
eprintln!(
|
|
"concurrency={concurrency:>3} ops={:>5} throughput={:>9.1} ops/s | acquire p50/p95/p99={}/{}/{}us | confirm p50/p95/p99={}/{}/{}us | release p50/p95/p99={}/{}/{}us | rss_delta_kib={}",
|
|
stats.ops,
|
|
throughput,
|
|
stats.acquire_p50,
|
|
stats.acquire_p95,
|
|
stats.acquire_p99,
|
|
stats.confirm_p50,
|
|
stats.confirm_p95,
|
|
stats.confirm_p99,
|
|
stats.release_p50,
|
|
stats.release_p95,
|
|
stats.release_p99,
|
|
rss_delta_kib,
|
|
);
|
|
}
|
|
|
|
// Soak-style churn: every connection and lease must return to baseline.
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
let usage = manager.provider_lease_usage(&input_name);
|
|
assert_eq!(usage.total(), 0, "lease table must return to baseline after churn");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn closing_state_blocks_slot_release_until_completion_token_cancelled() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager));
|
|
let input_name: Arc<str> = "provider_1".intern();
|
|
|
|
let handle1 = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([127, 0, 0, 1], 46_001)),
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some("session-1"),
|
|
)
|
|
.expect("first allocation should succeed");
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
let completion_token = handle1.completion_token.clone().expect("completion token present");
|
|
let alloc_id = handle1.allocation_id;
|
|
|
|
// Mark Closing (simulating supersede / release_playback_connections_await)
|
|
manager.mark_closing(alloc_id);
|
|
|
|
// Drop the handle while completion_token is still pending (upstream body not yet closed)
|
|
manager.release_handle(&handle1);
|
|
|
|
// The slot must still be occupied by the Closing allocation
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
// A second start attempt must fail because the provider is at its limit (1 connection)
|
|
let handle2 = manager.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([127, 0, 0, 1], 46_002)),
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some("session-2"),
|
|
);
|
|
assert!(handle2.is_none(), "second start must be rejected while first slot is Closing");
|
|
|
|
// Now signal completion (upstream body owner dropped upstream socket)
|
|
completion_token.cancel();
|
|
|
|
// Allow background reaper task to execute complete_release
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
|
|
// Second start attempt now succeeds
|
|
let handle3 = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([127, 0, 0, 1], 46_003)),
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some("session-3"),
|
|
)
|
|
.expect("start must succeed after completion releases the slot");
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
manager.release_handle(&handle3);
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reaper_cleanup_is_idempotent_and_removes_all_indices() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager));
|
|
let input_name: Arc<str> = "provider_1".intern();
|
|
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&SocketAddr::from(([127, 0, 0, 1], 47_001)),
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some("session-idempotent"),
|
|
)
|
|
.expect("allocation should succeed");
|
|
|
|
let completion_token = handle.completion_token.clone().expect("completion token present");
|
|
let alloc_id = handle.allocation_id;
|
|
manager.mark_closing(alloc_id);
|
|
|
|
manager.release_handle(&handle);
|
|
completion_token.cancel();
|
|
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
|
|
// Duplicate releases must be completely safe and no-op
|
|
manager.complete_release(alloc_id);
|
|
manager.release_handle(&handle);
|
|
manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 47_001)));
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_single_request_preemption_awaits_victim_completion() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager));
|
|
let input_name: Arc<str> = "provider_1".intern();
|
|
|
|
let low_addr = SocketAddr::from(([127, 0, 0, 1], 48_001));
|
|
let high_addr = SocketAddr::from(([127, 0, 0, 1], 48_002));
|
|
|
|
let low_handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&low_addr,
|
|
false,
|
|
10,
|
|
ConnectionKind::Normal,
|
|
Some("session-victim"),
|
|
)
|
|
.expect("initial low-priority allocation succeeds");
|
|
|
|
manager.mark_opening(low_handle.allocation_id);
|
|
assert!(manager.register_body_owner(low_handle.allocation_id));
|
|
|
|
let cancel_token = low_handle.cancel_token.clone().unwrap();
|
|
let completion_token = low_handle.completion_token.clone().unwrap();
|
|
|
|
// Simulate upstream body owner: when cancelled, complete release after 30ms
|
|
let comp_clone = completion_token.clone();
|
|
tokio::spawn(async move {
|
|
cancel_token.cancelled().await;
|
|
tokio::time::sleep(Duration::from_millis(30)).await;
|
|
comp_clone.cancel();
|
|
});
|
|
|
|
// High priority acquire should preempt the victim, await its completion, and succeed in the same call
|
|
let high_handle = manager
|
|
.acquire_connection_with_lease_for_session_await(
|
|
&input_name,
|
|
&high_addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
None,
|
|
)
|
|
.await
|
|
.expect("high-priority allocation must succeed in the same call after awaiting victim completion");
|
|
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
manager.release_handle(&high_handle);
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn stale_handle_does_not_cancel_newer_generation_slot() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager));
|
|
let input_name: Arc<str> = "provider_1".intern();
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 49_001));
|
|
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some("session-gen"),
|
|
)
|
|
.expect("initial allocation succeeds");
|
|
|
|
manager.mark_opening(handle.allocation_id);
|
|
assert!(manager.register_body_owner(handle.allocation_id));
|
|
|
|
// Gen 0 has an old completion token
|
|
let old_completion_token = handle.completion_token.clone().expect("completion token present");
|
|
|
|
// Renew opening tokens to advance generation to 1
|
|
let (_new_cancel, new_completion, gen) =
|
|
manager.renew_opening_tokens(handle.allocation_id).expect("renewal should succeed");
|
|
assert_eq!(gen, 1);
|
|
|
|
// Simulate an old handle belonging to generation 0 whose completion token was cancelled
|
|
old_completion_token.cancel();
|
|
let stale_handle = tuliprox_core::model::ProviderHandle {
|
|
playback_request_id: handle.playback_request_id,
|
|
binding_tag: handle.binding_tag,
|
|
client_id: handle.client_id,
|
|
allocation_id: handle.allocation_id,
|
|
allocation: handle.allocation.clone(),
|
|
cancel_token: handle.cancel_token.clone(),
|
|
completion_token: Some(old_completion_token),
|
|
close_reason: Arc::clone(&handle.close_reason),
|
|
open_generation: 0, // Stale generation
|
|
};
|
|
|
|
// Releasing the stale handle must NOT poison/cancel the new generation's completion token
|
|
manager.release_handle(&stale_handle);
|
|
assert!(
|
|
!new_completion.is_cancelled(),
|
|
"new generation completion token must remain uncancelled after stale handle release"
|
|
);
|
|
assert_eq!(manager.get_provider_connections_count(), 1, "slot must still be held by the active new generation");
|
|
|
|
// When the real new generation completes, the slot is reaped
|
|
new_completion.cancel();
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
assert_eq!(manager.get_provider_connections_count(), 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dropped_cleanup_future_does_not_permanently_leak_slot() {
|
|
let app_cfg = create_test_app_config_single_provider_pool();
|
|
let event_manager = Arc::new(EventManager::new());
|
|
let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager));
|
|
let input_name: Arc<str> = "provider_1".intern();
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], 49_002));
|
|
let addrs = [addr];
|
|
|
|
let handle = manager
|
|
.acquire_connection_with_grace_for_session(
|
|
&input_name,
|
|
&addr,
|
|
false,
|
|
0,
|
|
ConnectionKind::Normal,
|
|
Some("session-drop"),
|
|
)
|
|
.expect("allocation succeeds");
|
|
|
|
manager.mark_opening(handle.allocation_id);
|
|
assert!(manager.register_body_owner(handle.allocation_id));
|
|
let completion_token = handle.completion_token.clone().expect("token present");
|
|
|
|
// Start release_playback_connections_await but drop it before completion
|
|
{
|
|
let cleanup_fut = manager.release_playback_connections_await("session-drop", &addrs);
|
|
// Poll once and drop
|
|
tokio::select! {
|
|
biased;
|
|
() = async {} => {},
|
|
() = cleanup_fut => {},
|
|
}
|
|
}
|
|
|
|
// Slot must still be occupied
|
|
assert_eq!(manager.get_provider_connections_count(), 1);
|
|
|
|
// Now the handle is released and the body owner signals completion
|
|
manager.release_handle(&handle);
|
|
completion_token.cancel();
|
|
|
|
// Give the background reaper task time to run
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
|
|
assert_eq!(
|
|
manager.get_provider_connections_count(),
|
|
0,
|
|
"slot must be freed after completion and not permanently leaked in Closing state"
|
|
);
|
|
}
|
|
}
|