use super::{ build_rewrite_secret_fingerprint, safe_proxy_session_id, safe_session_key, GarbageCollectionPolicy, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsAccessLeaseLifecycleSnapshot, HlsAccessLeasePendingDeadline, HlsAccessLeaseSessionSnapshot, HlsAccessLeaseState, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsCacheMetrics, HlsExpiredSessionMarker, HlsExpiredSessionReason, HlsGarbageCollector, HlsLifecycleEvent, HlsLifecycleEventKey, HlsLifecycleManager, HlsMapWorkerPool, HlsOriginSource, HlsQosRegistry, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionHandle, HlsSessionKey, HlsSessionStore, HlsSessionStoreOutcome, ProxySessionId, SegmentFetchPolicy, TransientResourceStore, }; use crate::{ api::model::{ActiveProviderManager, ActiveUserManager, AppState}, model::{AppConfig, HlsCacheConfig, HlsManifestRecoveryBurstConfig, StripConfig}, }; use arc_swap::ArcSwap; use log::{debug, error, info}; use shared::utils::sanitize_sensitive_info; use std::{collections::HashMap, io, path::PathBuf, sync::Arc}; use tokio::sync::{RwLock, Semaphore}; use tokio_util::sync::CancellationToken; /// Root runtime object for the future HLS cache proxy. pub struct HlsProxyManager { sessions: Arc, segment_cache: Arc, segment_repair: Arc, segment_worker_pool: Arc, map_worker_pool: Arc, runtime_config: ArcSwap, transient_resources: Arc, access_leases: Arc>, lifecycle: Arc, account_overlap_cooldowns: Arc>>, metrics: Arc, qos: Arc, gc: Arc, } #[derive(Debug, Clone)] struct HlsProxyRuntimeConfig { enabled: bool, segment_fetch_policy: SegmentFetchPolicy, cache_duration_seconds: u64, strip: StripConfig, origin_manifest_timeout_ms: u64, manifest_recovery_burst: HlsManifestRecoveryBurstConfig, transient_resource_ttl_ms: u64, gc_policy: GarbageCollectionPolicy, rewrite_secret_fingerprint: String, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct HlsAccountOverlapCooldownKey { input_name: Arc, account_name: Arc, } #[derive(Debug, Clone, Copy)] struct HlsAccountOverlapCooldown { until_ms: u64, } #[derive(Debug, Clone, Copy)] enum HlsAccountOverlapCooldownReason { ReclaimedByOriginalOwner, SpeculativePromoted, } impl HlsAccountOverlapCooldownReason { fn as_log_reason(self) -> &'static str { match self { Self::ReclaimedByOriginalOwner => "reclaimed-by-original-owner", Self::SpeculativePromoted => "speculative-promoted", } } } fn hls_pending_manifest_follow_up_window_ms(target_duration: Option) -> u64 { let target_duration_secs = u64::from(target_duration.unwrap_or(15)).max(1); target_duration_secs.saturating_mul(2_000).max(10_000) } fn hls_pending_manifest_follow_up_deadline(now_ms: u64, target_duration: Option) -> HlsAccessLeasePendingDeadline { HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: now_ms.saturating_add(hls_pending_manifest_follow_up_window_ms(target_duration)), } } impl HlsProxyRuntimeConfig { fn from_config(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { Self::from_config_with_enabled(config, rewrite_secret, true) } fn from_config_with_enabled(config: &HlsCacheConfig, rewrite_secret: &[u8], enabled: bool) -> Self { Self { enabled, segment_fetch_policy: SegmentFetchPolicy::from_config(config), cache_duration_seconds: config.cache_duration, strip: config.strip.clone(), origin_manifest_timeout_ms: config.origin_manifest_timeout_ms, manifest_recovery_burst: config.manifest_recovery_burst.clone(), transient_resource_ttl_ms: config.cache_duration.saturating_mul(1_000), gc_policy: GarbageCollectionPolicy::from_config(config), rewrite_secret_fingerprint: build_rewrite_secret_fingerprint(rewrite_secret), } } } #[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] struct HlsProxySessionCleanupStats { access_leases: usize, repair_windows: usize, repair_generations: usize, repair_candidates: usize, repair_object_metadata: usize, repair_watchdog_metadata: usize, repair_watchdog_locks: usize, qos_access_leases: usize, } impl HlsProxySessionCleanupStats { fn did_cleanup(self) -> bool { self.access_leases > 0 || self.repair_windows > 0 || self.repair_generations > 0 || self.repair_candidates > 0 || self.repair_object_metadata > 0 || self.repair_watchdog_metadata > 0 || self.repair_watchdog_locks > 0 || self.qos_access_leases > 0 } } impl HlsProxyManager { pub fn new() -> Self { let default_dto = shared::model::HlsCacheConfigDto::default(); let default_config = HlsCacheConfig::from(&default_dto); Self::with_hls_cache_config(&default_config) } pub fn from_hls_cache_config(config: Option<&HlsCacheConfig>) -> Self { Self::from_hls_cache_config_and_secret(config, &[]) } pub fn from_hls_cache_config_and_secret(config: Option<&HlsCacheConfig>, rewrite_secret: &[u8]) -> Self { let default_config; let (config, enabled) = if let Some(config) = config { (config, true) } else { default_config = HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default()); (&default_config, false) }; Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, enabled) } pub fn with_cache_settings(cache_path: impl Into, cache_duration_seconds: u64) -> Self { let default_dto = shared::model::HlsCacheConfigDto { cache_duration: cache_duration_seconds, cache_path: Some(cache_path.into().to_string_lossy().to_string()), ..Default::default() }; let default_config = HlsCacheConfig::from(&default_dto); let segment_fetch_policy = SegmentFetchPolicy::from_config(&default_config); let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); let sessions = Arc::new(HlsSessionStore::new()); let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&default_config.cache_path))); segment_cache.update_cache_limits(default_config.cache_bytes, default_config.cache_bytes_per_session); let segment_repair = Arc::new(HlsSegmentRepairManager::new(default_config.segment_repair.clone())); let metrics = Arc::new(HlsCacheMetrics::default()); let qos = Arc::new(HlsQosRegistry::default()); let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); let lifecycle = Arc::new(HlsLifecycleManager::new()); let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); let gc_policy = GarbageCollectionPolicy::from_config(&default_config); let runtime_config = HlsProxyRuntimeConfig::from_config(&default_config, &[]); let gc = Arc::new(HlsGarbageCollector::new_with_metrics( Arc::clone(&sessions), Arc::clone(&segment_cache), gc_policy.clone(), runtime_config.rewrite_secret_fingerprint.clone(), Arc::clone(&metrics), )); Self { sessions, segment_cache, segment_repair, segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_and_metrics( segment_fetch_policy.clone(), Arc::clone(&global_fetch_semaphore), Arc::clone(&access_leases), Arc::clone(&metrics), )), map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_and_access_leases( segment_fetch_policy.clone(), global_fetch_semaphore, Arc::clone(&access_leases), )), runtime_config: ArcSwap::from_pointee(runtime_config), transient_resources: Arc::new(TransientResourceStore::new()), access_leases, lifecycle, account_overlap_cooldowns, metrics, qos, gc, } } pub fn with_hls_cache_config(config: &HlsCacheConfig) -> Self { Self::with_hls_cache_config_and_secret(config, &[]) } pub fn with_hls_cache_config_and_secret(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, true) } fn with_hls_cache_config_and_secret_enabled(config: &HlsCacheConfig, rewrite_secret: &[u8], enabled: bool) -> Self { let segment_fetch_policy = SegmentFetchPolicy::from_config(config); let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); let sessions = Arc::new(HlsSessionStore::new()); let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&config.cache_path))); segment_cache.update_cache_limits(config.cache_bytes, config.cache_bytes_per_session); let segment_repair = Arc::new(HlsSegmentRepairManager::new(config.segment_repair.clone())); let metrics = Arc::new(HlsCacheMetrics::default()); let qos = Arc::new(HlsQosRegistry::default()); let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); let lifecycle = Arc::new(HlsLifecycleManager::new()); let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); let gc_policy = GarbageCollectionPolicy::from_config(config); let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(config, rewrite_secret, enabled); let gc = Arc::new(HlsGarbageCollector::new_with_metrics( Arc::clone(&sessions), Arc::clone(&segment_cache), gc_policy.clone(), runtime_config.rewrite_secret_fingerprint.clone(), Arc::clone(&metrics), )); Self { sessions, segment_cache, segment_repair, segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_and_metrics( segment_fetch_policy.clone(), Arc::clone(&global_fetch_semaphore), Arc::clone(&access_leases), Arc::clone(&metrics), )), map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_and_access_leases( segment_fetch_policy.clone(), global_fetch_semaphore, Arc::clone(&access_leases), )), runtime_config: ArcSwap::from_pointee(runtime_config), transient_resources: Arc::new(TransientResourceStore::new()), access_leases, lifecycle, account_overlap_cooldowns, metrics, qos, gc, } } pub fn sessions(&self) -> &Arc { &self.sessions } pub fn segment_cache(&self) -> &Arc { &self.segment_cache } pub fn segment_repair(&self) -> &Arc { &self.segment_repair } pub fn segment_worker_pool(&self) -> &Arc { &self.segment_worker_pool } pub fn map_worker_pool(&self) -> &Arc { &self.map_worker_pool } pub fn segment_fetch_policy(&self) -> SegmentFetchPolicy { self.runtime_config.load().segment_fetch_policy.clone() } pub fn is_enabled(&self) -> bool { self.runtime_config.load().enabled } pub fn cache_duration_seconds(&self) -> u64 { self.runtime_config.load().cache_duration_seconds } pub fn session_idle_timeout_ms(&self) -> u64 { self.runtime_config.load().gc_policy.session_idle_timeout_ms } pub fn strip(&self) -> StripConfig { self.runtime_config.load().strip.clone() } pub fn origin_manifest_timeout_ms(&self) -> u64 { self.runtime_config.load().origin_manifest_timeout_ms } pub fn manifest_recovery_burst(&self) -> HlsManifestRecoveryBurstConfig { self.runtime_config.load().manifest_recovery_burst.clone() } pub fn transient_resource_ttl_ms(&self) -> u64 { self.runtime_config.load().transient_resource_ttl_ms } pub fn transient_resources(&self) -> &Arc { &self.transient_resources } pub fn access_leases(&self) -> &Arc> { &self.access_leases } pub fn lifecycle(&self) -> &Arc { &self.lifecycle } pub fn metrics(&self) -> &Arc { &self.metrics } pub fn qos(&self) -> &Arc { &self.qos } pub fn garbage_collector(&self) -> &Arc { &self.gc } pub fn gc_policy(&self) -> GarbageCollectionPolicy { self.runtime_config.load().gc_policy.clone() } pub fn rewrite_secret_fingerprint(&self) -> String { self.runtime_config.load().rewrite_secret_fingerprint.clone() } pub async fn is_account_overlap_cooling_down( &self, input_name: &Arc, account_name: &Arc, now_ms: u64, ) -> bool { let key = HlsAccountOverlapCooldownKey { input_name: Arc::clone(input_name), account_name: Arc::clone(account_name) }; let mut cooldowns = self.account_overlap_cooldowns.write().await; let Some(cooldown) = cooldowns.get(&key).copied() else { return false; }; if now_ms >= cooldown.until_ms { cooldowns.remove(&key); return false; } true } pub async fn mark_account_overlap_reclaimed_cooldown( &self, input_name: Arc, account_name: Arc, now_ms: u64, hard_active_window_ms: u64, ) { self.mark_account_overlap_cooldown( input_name, account_name, now_ms, hard_active_window_ms, HlsAccountOverlapCooldownReason::ReclaimedByOriginalOwner, ) .await; } pub async fn mark_account_overlap_promoted_cooldown( &self, input_name: Arc, account_name: Arc, now_ms: u64, hard_active_window_ms: u64, ) { self.mark_account_overlap_cooldown( input_name, account_name, now_ms, hard_active_window_ms, HlsAccountOverlapCooldownReason::SpeculativePromoted, ) .await; } async fn mark_account_overlap_cooldown( &self, input_name: Arc, account_name: Arc, now_ms: u64, hard_active_window_ms: u64, reason: HlsAccountOverlapCooldownReason, ) { let until_ms = now_ms.saturating_add(hard_active_window_ms); if until_ms <= now_ms { return; } let key = HlsAccountOverlapCooldownKey { input_name, account_name }; self.account_overlap_cooldowns.write().await.insert(key.clone(), HlsAccountOverlapCooldown { until_ms }); debug!( "HLS account overlap cooldown set for input {} account {} until {} ms after {}", sanitize_sensitive_info(key.input_name.as_ref()), sanitize_sensitive_info(key.account_name.as_ref()), until_ms, reason.as_log_reason() ); } pub async fn update_config(&self, app_config: &AppConfig) { let (hls_config, rewrite_secret, enabled) = { let config = app_config.config.load(); let rewrite_secret = config .reverse_proxy .as_ref() .map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); let hls_config = config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()).cloned(); let enabled = hls_config.is_some(); let hls_config = hls_config.unwrap_or_else(|| HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default())); (hls_config, rewrite_secret, enabled) }; let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(&hls_config, &rewrite_secret, enabled); let cache_path_changed = self.segment_cache.update_cache_path(PathBuf::from(&hls_config.cache_path)); self.segment_cache.update_cache_limits(hls_config.cache_bytes, hls_config.cache_bytes_per_session); if cache_path_changed { self.clear_runtime_cache_state_for_cache_path_change().await; } for session in self.sessions.list_sessions().await { session .write() .await .configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth); } self.segment_repair.update_config(hls_config.segment_repair.clone()); let global_fetch_semaphore = Arc::new(Semaphore::new(runtime_config.segment_fetch_policy.max_global_segment_fetches)); self.segment_worker_pool .update_config(runtime_config.segment_fetch_policy.clone(), Arc::clone(&global_fetch_semaphore)); self.map_worker_pool.update_config(runtime_config.segment_fetch_policy.clone(), global_fetch_semaphore); self.gc.update_config(runtime_config.gc_policy.clone(), runtime_config.rewrite_secret_fingerprint.clone()); self.runtime_config.store(Arc::new(runtime_config)); } async fn clear_runtime_cache_state_for_cache_path_change(&self) { self.sessions.clear().await; let removed_leases = self.access_leases.write().await.clear(); let removed_qos = self.qos.clear().await; self.segment_repair.clear_runtime_state().await; debug!( "HLS cache runtime state cleared after cache path change: access_leases_removed={removed_leases} qos_access_leases_removed={removed_qos}" ); } pub async fn prepare_access_lease(&self, lease: HlsAccessLease) { self.access_leases.write().await.prepare_access_lease(lease.clone()); self.schedule_access_lease_validity(&lease).await; } pub async fn access_lease( &self, lease_id: &HlsAccessLeaseId, proxy_session_id: &ProxySessionId, now_ms: u64, ) -> Option { let (lease, still_stored) = { let mut access_leases = self.access_leases.write().await; let lease = access_leases.access_lease(lease_id, proxy_session_id, now_ms); let still_stored = access_leases.lease_state(lease_id, now_ms).is_some(); (lease, still_stored) }; if lease.is_none() && !still_stored { self.segment_repair.remove_access_lease_window(lease_id).await; self.qos.remove_access_lease(lease_id).await; } lease } pub async fn access_lease_response_snapshot( &self, lease_id: &HlsAccessLeaseId, proxy_session_id: &ProxySessionId, now_ms: u64, ) -> Option { self.access_leases.write().await.response_snapshot(lease_id, proxy_session_id, now_ms) } pub async fn mark_access_leases_channel_unavailable_for_session( &self, proxy_session_id: &ProxySessionId, now_ms: u64, reason: HlsAccessLeaseChannelUnavailableReason, ) -> usize { self.access_leases.write().await.mark_channel_unavailable_for_session(proxy_session_id, now_ms, reason) } pub async fn mark_access_lease_channel_unavailable( &self, lease_id: &HlsAccessLeaseId, now_ms: u64, reason: HlsAccessLeaseChannelUnavailableReason, ) -> bool { self.access_leases.write().await.mark_channel_unavailable_for_lease(lease_id, now_ms, reason) } pub async fn update_access_lease_origin_acquire_policy( &self, lease_id: &HlsAccessLeaseId, connection_kind: crate::api::model::ConnectionKind, priority: i8, ) -> Option { self.access_leases.write().await.update_origin_acquire_policy(lease_id, connection_kind, priority) } pub async fn activate_access_lease( &self, lease_id: &HlsAccessLeaseId, proxy_session_id: &ProxySessionId, now_ms: u64, timing: HlsAccessLeaseTiming, ) -> HlsAccessLeaseActivation { let activation = self.access_leases.write().await.activate_access_lease(lease_id, proxy_session_id, now_ms, timing); if let HlsAccessLeaseActivation::Activated { lease, previous_state } = &activation { if matches!(previous_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Idle) { self.segment_repair.start_access_lease_window(lease.lease_id.clone()).await; } self.schedule_access_lease_activity(lease).await; self.schedule_access_lease_validity(lease).await; } activation } pub async fn touch_manifest_access_lease( &self, lease_id: &HlsAccessLeaseId, proxy_session_id: &ProxySessionId, now_ms: u64, active_timing: Option, pending_deadline: Option, ttl_ms: u64, ) -> HlsAccessLeaseTouch { let touch = self.access_leases.write().await.touch_manifest_access_lease( lease_id, proxy_session_id, now_ms, active_timing, pending_deadline, ttl_ms, ); if let HlsAccessLeaseTouch::Touched { lease } = &touch { if lease.state == HlsAccessLeaseState::Activated { self.schedule_access_lease_activity(lease).await; } self.schedule_access_lease_validity(lease).await; } touch } pub async fn mark_pending_manifest_follow_up_for_lease( &self, lease_id: &HlsAccessLeaseId, proxy_session_id: &ProxySessionId, now_ms: u64, target_duration: Option, ) -> bool { let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); let lease = self.access_leases.write().await.mark_pending_manifest_follow_up_for_lease( lease_id, proxy_session_id, now_ms, deadline, ); if let Some(lease) = lease { self.schedule_access_lease_validity(&lease).await; debug!( "HLS pending manifest lease shortened after manifest response: lease={} proxy_session={}", super::safe_hls_access_lease_id(lease_id), safe_proxy_session_id(proxy_session_id) ); true } else { false } } pub async fn mark_pending_manifest_follow_up_for_session( &self, proxy_session_id: &ProxySessionId, now_ms: u64, target_duration: Option, ) -> usize { let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); let leases = self.access_leases.write().await.mark_pending_manifest_follow_up_for_session( proxy_session_id, now_ms, deadline, ); for lease in &leases { self.schedule_access_lease_validity(lease).await; } leases.len() } pub async fn touch_access_lease( &self, lease_id: &HlsAccessLeaseId, now_ms: u64, timing: HlsAccessLeaseTiming, ) -> bool { let lease = self.access_leases.write().await.touch_access_lease_snapshot(lease_id, now_ms, timing); if let Some(lease) = lease { self.schedule_access_lease_activity(&lease).await; self.schedule_access_lease_validity(&lease).await; true } else { false } } pub async fn active_access_lease_count_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> usize { self.access_leases.write().await.active_access_lease_count_for_session(proxy_session_id, now_ms) } pub async fn has_usable_access_lease_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> bool { self.access_leases.write().await.has_usable_access_lease_for_session(proxy_session_id, now_ms) } pub async fn access_lease_session_snapshot( &self, proxy_session_id: &ProxySessionId, now_ms: u64, ) -> HlsAccessLeaseSessionSnapshot { self.access_leases.write().await.session_snapshot(proxy_session_id, now_ms) } async fn access_lease_lifecycle_snapshot( &self, lease_id: &HlsAccessLeaseId, now_ms: u64, ) -> Option { self.access_leases.write().await.lifecycle_snapshot(lease_id, now_ms) } async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) { self.access_leases.write().await.remove_access_lease(lease_id); self.segment_repair.remove_access_lease_window(lease_id).await; self.qos.remove_access_lease(lease_id).await; } async fn cleanup_proxy_session_state( &self, proxy_session_id: &ProxySessionId, reason: &'static str, ) -> HlsProxySessionCleanupStats { let before = self.segment_repair.stats().await; let removed_leases = self.access_leases.write().await.remove_access_leases_for_session(proxy_session_id); let username = removed_leases.first().map(|lease| lease.username.clone()); self.sessions.update_expired_session_marker_username(proxy_session_id, username).await; let removed_lease_ids = removed_leases.iter().map(|lease| lease.lease_id.clone()).collect::>(); self.segment_repair.remove_proxy_session_state(proxy_session_id, &removed_lease_ids).await; let removed_qos = self.qos.remove_access_leases(&removed_lease_ids).await; let removed_qos = removed_qos.saturating_add(self.qos.remove_proxy_session_state(proxy_session_id).await); let after = self.segment_repair.stats().await; let stats = HlsProxySessionCleanupStats { access_leases: removed_lease_ids.len(), repair_windows: before.windows.saturating_sub(after.windows), repair_generations: before.generations.saturating_sub(after.generations), repair_candidates: before.checked_candidates.saturating_sub(after.checked_candidates), repair_object_metadata: before.object_metadata.saturating_sub(after.object_metadata), repair_watchdog_metadata: before.watchdog_metadata.saturating_sub(after.watchdog_metadata), repair_watchdog_locks: before.watchdog_locks.saturating_sub(after.watchdog_locks), qos_access_leases: removed_qos, }; if stats.did_cleanup() { debug!( "HLS proxy session state cleaned: session={} reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_object_metadata={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", safe_proxy_session_id(proxy_session_id), reason, stats.access_leases, stats.repair_windows, stats.repair_generations, stats.repair_candidates, stats.repair_object_metadata, stats.repair_watchdog_metadata, stats.repair_watchdog_locks, stats.qos_access_leases ); } stats } pub async fn expired_session_marker( &self, proxy_session_id: &ProxySessionId, now_ms: u64, ) -> Option { self.sessions .expired_session_marker(proxy_session_id, now_ms, self.session_idle_timeout_ms().saturating_mul(2).max(1)) .await } async fn cleanup_all_runtime_state(&self, reason: &'static str) { let removed_access_leases = self.access_leases.write().await.clear(); self.account_overlap_cooldowns.write().await.clear(); let removed_qos = self.qos.clear().await; let before = self.segment_repair.stats().await; self.segment_repair.clear_runtime_state().await; if removed_access_leases > 0 || before.windows > 0 || before.generations > 0 || before.checked_candidates > 0 || before.metadata > 0 || before.object_metadata > 0 || before.locks > 0 || before.watchdog_metadata > 0 || before.watchdog_locks > 0 || removed_qos > 0 { debug!( "HLS runtime state cleaned: reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", reason, removed_access_leases, before.windows, before.generations, before.checked_candidates, before.metadata, before.object_metadata, before.locks, before.watchdog_metadata, before.watchdog_locks, removed_qos ); } } async fn cleanup_after_garbage_collection(&self, report: &super::GarbageCollectionReport) { if report.secret_cache_invalidated { self.cleanup_all_runtime_state("secret-cache-invalidated").await; return; } for proxy_session_id in &report.removed_session_ids { self.cleanup_proxy_session_state(proxy_session_id, "gc-session-removed").await; } } async fn schedule_access_lease_activity(&self, lease: &HlsAccessLease) { if let Some(active_until_ms) = lease.active_until_ms { self.lifecycle .schedule( HlsLifecycleEventKey::AccessLeaseActive { lease_id: lease.lease_id.clone(), proxy_session_id: lease.proxy_session_id.clone(), }, active_until_ms, ) .await; } } async fn schedule_access_lease_validity(&self, lease: &HlsAccessLease) { let due_at_ms = if lease.state == HlsAccessLeaseState::Pending { lease.pending_deadline_ms().unwrap_or(lease.valid_until_ms) } else { lease.valid_until_ms }; self.lifecycle .schedule( HlsLifecycleEventKey::AccessLeaseValidity { lease_id: lease.lease_id.clone(), proxy_session_id: lease.proxy_session_id.clone(), }, due_at_ms, ) .await; } async fn schedule_access_lease_lifecycle_snapshot(&self, snapshot: &HlsAccessLeaseLifecycleSnapshot) { if snapshot.state == HlsAccessLeaseState::Activated { if let Some(active_until_ms) = snapshot.active_until_ms { self.lifecycle .schedule( HlsLifecycleEventKey::AccessLeaseActive { lease_id: snapshot.lease_id.clone(), proxy_session_id: snapshot.proxy_session_id.clone(), }, active_until_ms, ) .await; } } if snapshot.state != HlsAccessLeaseState::Expired && snapshot.state != HlsAccessLeaseState::Denied { let due_at_ms = if snapshot.state == HlsAccessLeaseState::Pending { snapshot.pending_deadline.map_or(snapshot.valid_until_ms, HlsAccessLeasePendingDeadline::deadline_ms) } else { snapshot.valid_until_ms }; self.lifecycle .schedule( HlsLifecycleEventKey::AccessLeaseValidity { lease_id: snapshot.lease_id.clone(), proxy_session_id: snapshot.proxy_session_id.clone(), }, due_at_ms, ) .await; } } pub async fn schedule_session_idle_for_handle(&self, session: &HlsSessionHandle) { let session_idle_timeout_ms = self.session_idle_timeout_ms(); let (proxy_session_id, due_at_ms) = { let session = session.read().await; (session.proxy_session_id.clone(), session.idle_expiry_due_at_ms(session_idle_timeout_ms)) }; self.lifecycle.schedule(HlsLifecycleEventKey::SessionIdle { proxy_session_id }, due_at_ms).await; } pub async fn mark_authorized_media_access_for_session(&self, session: &HlsSessionHandle, now_ms: u64) { session.write().await.mark_authorized_media_access(now_ms); self.schedule_session_idle_for_handle(session).await; } pub async fn handle_lifecycle_event( &self, active_users: &Arc, active_provider: &Arc, event: HlsLifecycleEvent, now_ms: u64, ) { if !self.is_enabled() { return; } match event.key { HlsLifecycleEventKey::AccessLeaseActive { lease_id, proxy_session_id } | HlsLifecycleEventKey::AccessLeaseValidity { lease_id, proxy_session_id } => { let mut should_sync_session = false; if let Some(snapshot) = self.access_lease_lifecycle_snapshot(&lease_id, now_ms).await { should_sync_session = true; if let Some(release) = &snapshot.idle_release { active_users .release_session_streams_and_counted_reservation( &release.username, &release.user_session_token, ) .await; debug!( "HLS access lease idled: lease={} proxy_session={} session={}", super::safe_hls_access_lease_id(&release.lease_id), safe_proxy_session_id(&snapshot.proxy_session_id), super::safe_user_session_token(&release.user_session_token) ); } if matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied) { self.remove_access_lease(&snapshot.lease_id).await; debug!( "HLS access lease removed: lease={} proxy_session={} state={}", super::safe_hls_access_lease_id(&snapshot.lease_id), safe_proxy_session_id(&snapshot.proxy_session_id), snapshot.state.as_log_value() ); debug!( "HLS lifecycle state snapshot: trigger=access-lease-removed {}", self.debug_state_summary().await ); } else { self.schedule_access_lease_lifecycle_snapshot(&snapshot).await; } } if should_sync_session { if let Some(session) = self.sessions.get_by_proxy_session_id(&proxy_session_id).await { self.sync_session_access_lease_count_and_detach_if_needed( active_users, active_provider, &session, &proxy_session_id, now_ms, ) .await; } } } HlsLifecycleEventKey::SessionIdle { proxy_session_id } => { self.handle_session_idle_lifecycle_event(&proxy_session_id, now_ms).await; } } } async fn handle_session_idle_lifecycle_event(&self, proxy_session_id: &ProxySessionId, now_ms: u64) { let Some(session) = self.sessions.get_by_proxy_session_id(proxy_session_id).await else { return; }; let session_idle_timeout_ms = self.session_idle_timeout_ms(); let (key, due_at_ms, can_remove) = { let session = session.read().await; ( session.key.clone(), session.idle_expiry_due_at_ms(session_idle_timeout_ms), session.can_expire_idle_session(now_ms, session_idle_timeout_ms), ) }; if !can_remove { self.lifecycle .schedule( HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, due_at_ms.max(now_ms.saturating_add(1)), ) .await; return; } if self.segment_cache.has_active_temp_files_for_session(proxy_session_id).await { self.lifecycle .schedule( HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, now_ms.saturating_add(1_000), ) .await; return; } let username = self.access_leases.read().await.first_username_for_session(proxy_session_id); if self .sessions .remove_session_marking_expired( &key, proxy_session_id, now_ms, HlsExpiredSessionReason::SessionIdleTimeout, username, ) .await .is_some() { self.cleanup_proxy_session_state(proxy_session_id, "lifecycle-session-expired").await; if let Err(err) = self.segment_cache.delete_session_dir(proxy_session_id).await { error!( "HLS session lifecycle cleanup failed: session={} error={err}", safe_proxy_session_id(proxy_session_id) ); } else { debug!("HLS session lifecycle expired: session={}", safe_proxy_session_id(proxy_session_id)); debug!("HLS lifecycle state snapshot: trigger=session-expired {}", self.debug_state_summary().await); } } } pub async fn debug_state_summary(&self) -> String { let sessions = self.sessions.list_sessions().await; let access_leases = self.access_leases.read().await.len(); let qos_access_leases = self.qos.len().await; let repair = self.segment_repair.stats().await; let mut segments = 0_usize; let mut maps = 0_usize; let mut transient_resources = 0_usize; let mut transient_objects = 0_usize; let mut active_origin_work = 0_usize; let mut active_segment_fetches = 0_usize; let mut active_map_fetches = 0_usize; for session in &sessions { let session = session.read().await; segments = segments.saturating_add(session.segments.len()); maps = maps.saturating_add(session.maps.len()); transient_resources = transient_resources.saturating_add(session.transient.resources.len()); transient_objects = transient_objects.saturating_add(session.transient.object_cache.len()); active_origin_work = active_origin_work.saturating_add(session.activity.active_origin_work_count); active_segment_fetches = active_segment_fetches.saturating_add(session.active_segment_fetches); active_map_fetches = active_map_fetches.saturating_add(session.active_map_fetches); } format!( "sessions={} access_leases={} qos_access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} segments={} maps={} transient_resources={} transient_objects={} active_origin_work={} active_segment_fetches={} active_map_fetches={}", sessions.len(), access_leases, qos_access_leases, repair.windows, repair.generations, repair.checked_candidates, repair.metadata, repair.object_metadata, repair.locks, repair.watchdog_metadata, repair.watchdog_locks, segments, maps, transient_resources, transient_objects, active_origin_work, active_segment_fetches, active_map_fetches ) } pub async fn run_garbage_collection_once(&self, now_ms: u64) -> io::Result { if !self.is_enabled() { return Ok(super::GarbageCollectionReport::default()); } let report = self.gc.run_once(now_ms).await?; self.cleanup_after_garbage_collection(&report).await; Ok(report) } pub async fn sync_session_access_lease_count_and_detach_if_needed( &self, active_users: &Arc, _active_provider: &Arc, session: &HlsSessionHandle, proxy_session_id: &ProxySessionId, now_ms: u64, ) { let snapshot = self.access_lease_session_snapshot(proxy_session_id, now_ms).await; for release in &snapshot.idle_releases { active_users .release_session_streams_and_counted_reservation(&release.username, &release.user_session_token) .await; debug!( "HLS access lease idled: lease={} proxy_session={} session={}", super::safe_hls_access_lease_id(&release.lease_id), safe_proxy_session_id(proxy_session_id), super::safe_user_session_token(&release.user_session_token) ); } { let mut session = session.write().await; session.activity.active_access_lease_count = snapshot.active_count; session.reconcile_effective_origin_acquire_policy(snapshot.effective_origin_policy, now_ms); } } pub async fn sync_all_session_access_leases_and_detach_if_needed( &self, active_users: &Arc, active_provider: &Arc, now_ms: u64, ) { if !self.is_enabled() { return; } for session in self.sessions.list_sessions().await { let proxy_session_id = session.read().await.proxy_session_id.clone(); self.sync_session_access_lease_count_and_detach_if_needed( active_users, active_provider, &session, &proxy_session_id, now_ms, ) .await; } } pub async fn deny_access_lease(&self, lease_id: &HlsAccessLeaseId) { self.access_leases.write().await.deny_access_lease(lease_id); } pub async fn get_or_create_session( &self, key: HlsSessionKey, reverse_proxy_rewrite_secret: &[u8], now_ms: u64, ) -> HlsSessionHandle { self.get_or_create_session_with_outcome(key, reverse_proxy_rewrite_secret, now_ms).await.0 } pub async fn get_or_create_session_with_outcome( &self, key: HlsSessionKey, reverse_proxy_rewrite_secret: &[u8], now_ms: u64, ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { let origin_source = HlsOriginSource::from_session_key(&key); self.get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) .await } pub async fn get_or_create_session_with_source_and_outcome( &self, key: HlsSessionKey, origin_source: HlsOriginSource, reverse_proxy_rewrite_secret: &[u8], now_ms: u64, ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { let (session, outcome) = self .sessions .get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) .await; let (proxy_session_id, session_key) = { let session_guard = session.read().await; (safe_proxy_session_id(&session_guard.proxy_session_id), safe_session_key(&session_guard.key)) }; match outcome { HlsSessionStoreOutcome::Created => { self.metrics.record_session_created(); info!("HLS session created: session={session_key} proxy_session_id={proxy_session_id}"); } HlsSessionStoreOutcome::Reused => { self.metrics.record_session_reused(); debug!("HLS session reused: session={session_key} proxy_session_id={proxy_session_id}"); } } self.schedule_session_idle_for_handle(&session).await; session.write().await.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth); (session, outcome) } } impl Default for HlsProxyManager { fn default() -> Self { Self::new() } } pub fn exec_hls_lifecycle(app_state: &Arc, cancel_token: &CancellationToken) { let hls_proxy = Arc::clone(&app_state.hls_proxy); let active_users = Arc::clone(&app_state.active_users); let active_provider = Arc::clone(&app_state.active_provider); let cancel_token = cancel_token.clone(); tokio::spawn(async move { while let Some(event) = hls_proxy.lifecycle().next_event(&cancel_token).await { let now_ms = current_time_millis(); hls_proxy.handle_lifecycle_event(&active_users, &active_provider, event, now_ms).await; } }); } fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } #[cfg(test)] mod tests { use super::HlsProxyManager; use crate::{ api::model::HlsSessionKey, model::{AppConfig, Config, HlsCacheConfig, MediaToolCapabilities, ReverseProxyConfig, SourcesConfig}, utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; use shared::model::{ ConfigPaths, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsStripConfigDto, HlsStripMode, ReverseProxyConfigDto, }; use std::sync::Arc; fn empty_paths() -> ConfigPaths { ConfigPaths { home_path: String::new(), config_path: String::new(), storage_path: String::new(), config_file_path: String::new(), sources_file_path: String::new(), mapping_file_path: None, mapping_files_used: None, template_file_path: None, template_files_used: None, api_proxy_file_path: String::new(), custom_stream_response_path: None, } } fn test_app_config(config: Config) -> AppConfig { AppConfig { config: Arc::new(ArcSwap::from_pointee(config)), sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), hdhomerun: Arc::new(ArcSwapOption::default()), api_proxy: Arc::new(ArcSwapOption::default()), file_locks: Arc::new(FileLockManager::default()), paths: Arc::new(ArcSwap::from_pointee(empty_paths())), custom_stream_response: Arc::new(ArcSwapOption::default()), access_token_secret: [0; 32], encrypt_secret: [7; 16], media_tools: Arc::new(MediaToolCapabilities::new()), } } fn config_with_hls_cache(hls_cache: HlsCacheConfigDto) -> Config { Config { reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { hls_cache: Some(hls_cache), ..ReverseProxyConfigDto::default() })), ..Config::default() } } #[tokio::test] async fn update_config_applies_hls_runtime_settings_to_existing_manager() { let initial_dto = HlsCacheConfigDto { cache_path: Some("/tmp/tuliprox/hls-a".to_string()), max_segments_prefetch: 1, ..Default::default() }; let initial_config = HlsCacheConfig::from(&initial_dto); let manager = HlsProxyManager::with_hls_cache_config(&initial_config); let (session, _) = manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; let mut updated_dto = initial_dto.clone(); updated_dto.max_segments_prefetch = 4; updated_dto.max_concurrent_segment_fetches_per_session = 5; updated_dto.max_concurrent_segment_fetches_global = 6; updated_dto.origin_manifest_timeout_ms = 1_234; updated_dto.origin_segment_timeout_ms = 5_678; updated_dto.cache_duration = 99; updated_dto.session_idle_timeout = 55; updated_dto.manifest_recovery_burst.level = shared::model::HlsManifestRecoveryBurstLevel::Balanced; updated_dto.strip = HlsStripConfigDto { mode: HlsStripMode::Seconds, value: 7 }; let app_config = test_app_config(config_with_hls_cache(updated_dto)); manager.update_config(&app_config).await; assert_eq!(manager.segment_fetch_policy().max_prefetch_queue_depth, 4); assert_eq!(manager.segment_fetch_policy().max_session_segment_fetches, 5); assert_eq!(manager.segment_fetch_policy().max_global_segment_fetches, 6); assert_eq!(manager.segment_fetch_policy().origin_segment_timeout_ms, 5_678); assert_eq!(manager.origin_manifest_timeout_ms(), 1_234); assert_eq!(manager.cache_duration_seconds(), 99); assert_eq!(manager.transient_resource_ttl_ms(), 99_000); assert_eq!(manager.session_idle_timeout_ms(), 55_000); assert_eq!(manager.manifest_recovery_burst().level, HlsManifestRecoveryBurstLevel::Balanced); assert_eq!(manager.strip().mode, HlsStripMode::Seconds); assert_eq!(manager.strip().value, 7); assert_eq!(session.read().await.segment_prefetch_queue.max_prefetch_depth(), 4); } #[tokio::test] async fn optional_hls_config_controls_runtime_enabled_state() { let manager = HlsProxyManager::from_hls_cache_config(None); assert!(!manager.is_enabled()); assert_eq!( manager.run_garbage_collection_once(1_000).await.expect("disabled gc should no-op"), super::super::GarbageCollectionReport::default() ); let app_config = test_app_config(config_with_hls_cache(HlsCacheConfigDto::default())); manager.update_config(&app_config).await; assert!(manager.is_enabled()); } #[tokio::test] async fn update_config_cache_path_change_clears_runtime_cache_state() { let temp_dir = tempfile::tempdir().expect("temp dir"); let old_cache = temp_dir.path().join("old"); let new_cache = temp_dir.path().join("new"); let initial_dto = HlsCacheConfigDto { cache_path: Some(old_cache.to_string_lossy().to_string()), ..Default::default() }; let initial_config = HlsCacheConfig::from(&initial_dto); let manager = HlsProxyManager::with_hls_cache_config(&initial_config); let _ = manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; assert_eq!(manager.sessions().len().await, 1); let mut updated_dto = initial_dto; updated_dto.cache_path = Some(new_cache.to_string_lossy().to_string()); let app_config = test_app_config(config_with_hls_cache(updated_dto)); manager.update_config(&app_config).await; assert!(manager.sessions().is_empty().await); assert_eq!(manager.segment_cache().cache_path(), new_cache); } }