From e914cbd68eef1edcd606fc97e28c705a54164e0b Mon Sep 17 00:00:00 2001 From: euzu Date: Wed, 14 Oct 2026 19:23:35 +0200 Subject: [PATCH] feature stream failover --- Cargo.lock | 2 + .../app/src/api/api_utils/channel_group.rs | 372 +++++++ backend/app/src/api/api_utils/mod.rs | 6 +- .../app/src/api/api_utils/provider_acquire.rs | 664 +++++++++++-- backend/app/src/api/api_utils/streaming.rs | 37 +- backend/app/src/api/api_utils/tests.rs | 5 + .../tests/admission_session_admission.rs | 5 +- .../api_utils/tests/channel_group_playback.rs | 127 +++ .../tests/channel_group_selection.rs | 234 +++++ .../api_utils/tests/channel_group_sticky.rs | 70 ++ .../app/src/api/api_utils/tests/cleanup.rs | 11 +- .../api_utils/tests/group_playback_support.rs | 933 ++++++++++++++++++ .../src/api/api_utils/tests/maintenance.rs | 5 +- .../src/api/api_utils/tests/provider_urls.rs | 1 + .../stream_responses_response_headers.rs | 5 +- .../stream_responses_session_activation.rs | 5 +- .../app/src/api/api_utils/tests/support.rs | 25 + .../api/endpoints/custom_video_stream_api.rs | 1 + .../api/endpoints/hls_api/manifest/cached.rs | 8 +- .../hls_api/manifest/provisioning.rs | 45 +- .../src/api/endpoints/hls_api/owner_token.rs | 50 +- .../api/endpoints/hls_api/segment/entry.rs | 351 +++++-- .../endpoints/hls_api/segment/registration.rs | 1 + .../api/endpoints/hls_api/segment/routing.rs | 74 +- .../app/src/api/endpoints/hls_api/tests.rs | 2 + .../endpoints/hls_api/tests/channel_group.rs | 38 + .../hls_api/tests/support/app_state.rs | 1 + backend/app/src/api/endpoints/m3u_api.rs | 148 ++- backend/app/src/api/endpoints/v1_api.rs | 1 + .../endpoints/v1_api_playlist/recording.rs | 1 + .../api/endpoints/v1_api_playlist/tests.rs | 1 + .../v1_api_playlist/tests/channel_groups.rs | 101 ++ .../v1_api_playlist/tests/support.rs | 1 + .../endpoints/v1_api_playlist/webplayer.rs | 2 +- .../app/src/api/endpoints/websocket_api.rs | 22 +- .../app/src/api/endpoints/xtream_api/mod.rs | 143 ++- .../app/src/api/endpoints/xtream_api/tests.rs | 1 + backend/app/src/api/main_api.rs | 79 +- backend/app/src/api/model/app_state/mod.rs | 31 + backend/app/src/api/model/app_state/reload.rs | 10 +- .../src/api/model/app_state/test_support.rs | 1 + .../active_client_stream/tests/admission.rs | 1 + .../active_client_stream/tests/support.rs | 2 + .../model/streams/provider_stream_factory.rs | 3 +- .../provider_stream_factory/opening.rs | 4 +- .../provider_stream_factory/options.rs | 64 +- .../provider_stream_factory/request.rs | 32 +- .../tests/streaming.rs | 1 + backend/app/src/api/serve.rs | 1 + .../app/src/api/tasks/notification_bridge.rs | 17 +- backend/core/src/model/config/target.rs | 13 +- backend/core/src/model/provider.rs | 32 + backend/iptv/src/error.rs | 1 + .../metadata/src/manager/task_processing.rs | 14 +- backend/mpegts/Cargo.toml | 4 + backend/mpegts/src/lib.rs | 71 ++ .../src/processor/channel_group/assemble.rs | 543 ++++++++++ .../src/processor/channel_group/mod.rs | 5 + .../src/processor/channel_group/tests.rs | 276 ++++++ .../processing/src/processor/deduplicate.rs | 84 +- backend/processing/src/processor/match_key.rs | 193 ++++ backend/processing/src/processor/mod.rs | 2 + .../processing/src/processor/playlist/mod.rs | 4 +- .../src/processor/playlist/target.rs | 78 +- .../src/processor/playlist/tests.rs | 2 + .../playlist/tests/channel_groups.rs | 89 ++ .../playlist/tests/curation_effect_gate.rs | 1 + .../playlist/tests/curation_selection.rs | 2 +- .../playlist/tests/identity_serialization.rs | 7 +- .../processing/src/processor/stream_probe.rs | 20 +- .../src/channel_group_repository.rs | 616 ++++++++++++ backend/repository/src/lib.rs | 4 + backend/repository/src/playlist_mem_cache.rs | 22 +- backend/repository/src/playlist_repository.rs | 1 + .../src/playlist_repository/episode.rs | 62 +- .../src/playlist_repository/persist.rs | 159 ++- .../src/playlist_repository/tests.rs | 1 + .../tests/channel_groups.rs | 341 +++++++ backend/repository/src/storage_const.rs | 4 + backend/repository/src/stream_history/mod.rs | 1 + backend/repository/src/target_stream_item.rs | 423 ++++++++ backend/session/Cargo.toml | 2 + .../session/src/active_provider_manager.rs | 6 +- .../src/active_provider_manager/allocation.rs | 4 + .../active_provider_manager/capacity_probe.rs | 145 +++ .../src/active_provider_manager/tests.rs | 1 + .../tests/capacity_probe.rs | 105 ++ .../src/active_user_manager/activity.rs | 15 +- .../src/active_user_manager/admission.rs | 16 + .../session/src/active_user_manager/mod.rs | 9 + .../src/active_user_manager/observability.rs | 20 + .../active_user_manager/session_lifecycle.rs | 1 + .../tests/observability.rs | 1 + .../src/active_user_manager/tests/support.rs | 2 + backend/session/src/channel_group/circuit.rs | 299 ++++++ backend/session/src/channel_group/health.rs | 525 ++++++++++ backend/session/src/channel_group/mod.rs | 9 + backend/session/src/channel_group/ranking.rs | 441 +++++++++ .../session/src/channel_group/test_support.rs | 18 + .../session/src/connection_manager/tests.rs | 1 + backend/session/src/event_manager.rs | 24 + backend/session/src/lib.rs | 7 +- .../session/src/provider_lineup_manager.rs | 133 +++ .../session/src/qos_aggregation_manager.rs | 21 +- .../streams/shared_stream_manager/tests.rs | 1 + .../dashboard/stream_display/helpers.rs | 1 + .../dashboard/stream_display/mod.rs | 1 + frontend/src/hooks/use_server_status.rs | 2 + shared/src/error/tuliprox_error.rs | 1 + shared/src/model/channel_group.rs | 152 +++ shared/src/model/config/target.rs | 8 + .../src/model/config/target/channel_groups.rs | 276 ++++++ shared/src/model/config/target/options.rs | 10 + shared/src/model/event/kind.rs | 16 +- shared/src/model/event/message.rs | 65 +- shared/src/model/event/sink.rs | 20 + shared/src/model/item_field.rs | 3 + shared/src/model/mapping.rs | 1 + shared/src/model/media_properties.rs | 48 +- shared/src/model/mod.rs | 25 +- shared/src/model/playlist/header.rs | 10 + shared/src/model/playlist/item.rs | 6 + shared/src/model/playlist/m3u.rs | 6 +- shared/src/model/playlist/tests/identity.rs | 21 + shared/src/model/playlist/tests/m3u.rs | 3 +- shared/src/model/stream_info.rs | 9 +- 126 files changed, 8813 insertions(+), 497 deletions(-) create mode 100644 backend/app/src/api/api_utils/channel_group.rs create mode 100644 backend/app/src/api/api_utils/tests/channel_group_playback.rs create mode 100644 backend/app/src/api/api_utils/tests/channel_group_selection.rs create mode 100644 backend/app/src/api/api_utils/tests/channel_group_sticky.rs create mode 100644 backend/app/src/api/api_utils/tests/group_playback_support.rs create mode 100644 backend/app/src/api/endpoints/hls_api/tests/channel_group.rs create mode 100644 backend/app/src/api/endpoints/v1_api_playlist/tests/channel_groups.rs create mode 100644 backend/processing/src/processor/channel_group/assemble.rs create mode 100644 backend/processing/src/processor/channel_group/mod.rs create mode 100644 backend/processing/src/processor/channel_group/tests.rs create mode 100644 backend/processing/src/processor/match_key.rs create mode 100644 backend/processing/src/processor/playlist/tests/channel_groups.rs create mode 100644 backend/repository/src/channel_group_repository.rs create mode 100644 backend/repository/src/playlist_repository/tests/channel_groups.rs create mode 100644 backend/repository/src/target_stream_item.rs create mode 100644 backend/session/src/active_provider_manager/capacity_probe.rs create mode 100644 backend/session/src/active_provider_manager/tests/capacity_probe.rs create mode 100644 backend/session/src/channel_group/circuit.rs create mode 100644 backend/session/src/channel_group/health.rs create mode 100644 backend/session/src/channel_group/mod.rs create mode 100644 backend/session/src/channel_group/ranking.rs create mode 100644 backend/session/src/channel_group/test_support.rs create mode 100644 shared/src/model/channel_group.rs create mode 100644 shared/src/model/config/target/channel_groups.rs diff --git a/Cargo.lock b/Cargo.lock index 50bfcf93e..2e5308d7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4917,6 +4917,8 @@ dependencies = [ "lru", "rand 0.9.5", "reqwest", + "serde", + "serde_json", "shared", "tempfile", "thiserror 2.0.20", diff --git a/backend/app/src/api/api_utils/channel_group.rs b/backend/app/src/api/api_utils/channel_group.rs new file mode 100644 index 000000000..4bf7e1853 --- /dev/null +++ b/backend/app/src/api/api_utils/channel_group.rs @@ -0,0 +1,372 @@ +#![allow(dead_code)] + +use crate::{api::model::AppState, model::ConfigInput}; +use shared::model::{ChannelGroupPolicy, ChannelGroupSelectionConfig, StreamChannel}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_repository::channel_group_repository::{ChannelGroupRecord, GroupMember}; +pub(crate) use tuliprox_session::channel_group::{member_key, CircuitAdmission, CircuitView, SelectionMode}; +use tuliprox_session::{ + active_provider_manager::{CapacityTier, ConnectionKind, ManagedProviderHandle, PlaybackLeaseRef}, + channel_group::{rank_members, HalfOpenGuard, RankedCandidate}, +}; + +pub(crate) enum PlaybackSource<'a> { + Single { input: &'a Arc, stream_url: &'a str }, + Group { group: Arc, mode: SelectionMode, prefer: Option }, +} + +pub(crate) struct ResolvedGroupPlayback { + pub acquired: AcquiredMember, + pub member: GroupMember, + pub input: Arc, + pub stream_url: Arc, + pub stream_channel: StreamChannel, +} + +pub(crate) fn group_shared_key(target_id: u16, group_vid: u32) -> Arc { + Arc::from(format!("group:{target_id}:{group_vid}")) +} + +pub(crate) struct GroupSelectionRequest<'a> { + pub target_name: &'a Arc, + pub group: &'a ChannelGroupRecord, + pub policy: ChannelGroupPolicy, + pub mode: SelectionMode, + pub selection: ChannelGroupSelectionConfig, + pub addr: &'a SocketAddr, + pub username: &'a str, + pub priority: i8, + pub kind: ConnectionKind, + pub lease: Option>, + pub allow_provider_grace: bool, + pub exclude: &'a [usize], + pub demoted_inputs: &'a [Arc], + pub prefer: Option, +} + +pub(crate) struct AcquiredMember { + pub index: usize, + pub handle: ManagedProviderHandle, + pub tier: CapacityTier, + pub half_open: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum GroupSelectionFailure { + Exhausted, + NoCandidate, +} + +pub(crate) async fn acquire_group_member( + app_state: &Arc, + req: GroupSelectionRequest<'_>, +) -> Result { + acquire_group_member_inner(app_state, req, None::)>).await +} + +#[cfg(test)] +pub(crate) async fn acquire_group_member_with_race)>( + app_state: &Arc, + req: GroupSelectionRequest<'_>, + after_probe: F, +) -> Result { + acquire_group_member_inner(app_state, req, Some(after_probe)).await +} + +async fn acquire_group_member_inner)>( + app_state: &Arc, + req: GroupSelectionRequest<'_>, + after_probe: Option, +) -> Result { + let budget_secs = req.selection.startup_budget_secs; + if budget_secs > 0 { + match tokio::time::timeout(Duration::from_secs(budget_secs), select_member_loop(app_state, req, after_probe)) + .await + { + Ok(result) => result, + Err(_) => Err(GroupSelectionFailure::Exhausted), + } + } else { + select_member_loop(app_state, req, after_probe).await + } +} + +fn check_member_circuit( + app_state: &Arc, + req: &GroupSelectionRequest<'_>, + cand_idx: usize, + first_remaining_idx: usize, + all_circuits_open: bool, +) -> (bool, Option) { + let key = member_key(&req.group.members[cand_idx]); + let admission = app_state.member_health.admit(&key, std::time::Instant::now()); + match admission { + CircuitAdmission::Closed => (true, None), + CircuitAdmission::HalfOpen(guard) => (true, Some(guard)), + CircuitAdmission::Open => (all_circuits_open && cand_idx == first_remaining_idx, None), + } +} + +fn select_tier_candidates( + app_state: &Arc, + req: &GroupSelectionRequest<'_>, + remaining: &mut Vec, + probed_tiers: &mut Vec<(usize, CapacityTier)>, + all_circuits_open: bool, + filter_tier: F, + probe_tier: &dyn Fn(usize) -> CapacityTier, +) -> Option +where + F: Fn(CapacityTier) -> Option, +{ + loop { + let mut tried_candidate = false; + let candidate_indices: Vec<(usize, bool)> = probed_tiers + .iter() + .filter_map(|(idx, tier)| filter_tier(*tier).map(|allow_grace| (*idx, allow_grace))) + .collect(); + + for (cand_idx, allow_grace) in candidate_indices { + let Some((_, tier)) = probed_tiers.iter().find(|(idx, _)| *idx == cand_idx).copied() else { + continue; + }; + + let (can_try, half_open_guard) = + check_member_circuit(app_state, req, cand_idx, remaining[0].index, all_circuits_open); + if !can_try { + continue; + } + + tried_candidate = true; + let handle_opt = app_state.active_provider.try_acquire_connection_no_preempt( + &req.group.members[cand_idx].input_name, + allow_grace, + req.addr, + req.priority, + req.kind, + req.lease, + ); + + if let Some(handle) = handle_opt { + return Some(AcquiredMember { + index: cand_idx, + handle: ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle), + tier, + half_open: half_open_guard, + }); + } + + remaining.retain(|c| c.index != cand_idx); + *probed_tiers = remaining.iter().map(|c| (c.index, probe_tier(c.index))).collect(); + break; + } + + if !tried_candidate { + break; + } + } + None +} + +async fn try_acquire_preempt( + app_state: &Arc, + req: &GroupSelectionRequest<'_>, + remaining: &[RankedCandidate], + all_circuits_open: bool, +) -> Option { + for c in remaining { + let (can_try, half_open_guard) = + check_member_circuit(app_state, req, c.index, remaining[0].index, all_circuits_open); + if !can_try { + continue; + } + + let handle_opt = app_state + .active_provider + .acquire_connection_with_lease_for_session_await( + &req.group.members[c.index].input_name, + req.addr, + req.allow_provider_grace, + req.priority, + req.kind, + req.lease, + ) + .await; + + if let Some(handle) = handle_opt { + return Some(AcquiredMember { + index: c.index, + handle: ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle), + tier: CapacityTier::Preempt, + half_open: half_open_guard, + }); + } + + break; + } + None +} + +async fn select_member_loop)>( + app_state: &Arc, + req: GroupSelectionRequest<'_>, + after_probe: Option, +) -> Result { + let now = std::time::Instant::now(); + let mut ranked = + rank_members(req.target_name, &req.group.members, req.policy, &app_state.member_health, &req.mode, now); + + ranked.retain(|c| !req.exclude.contains(&c.index)); + if ranked.is_empty() { + return Err(GroupSelectionFailure::NoCandidate); + } + + ranked.sort_by_key(|c| req.demoted_inputs.contains(&req.group.members[c.index].input_name)); + + if let Some(prefer_idx) = req.prefer { + if let Some(pos) = ranked.iter().position(|c| c.index == prefer_idx) { + let cand = ranked.remove(pos); + ranked.insert(0, cand); + } + } + + let all_circuits_open = ranked.iter().all(|c| { + let key = member_key(&req.group.members[c.index]); + matches!(app_state.member_health.circuit_view(&key, now), CircuitView::Open { remaining } if remaining > Duration::ZERO) + }); + + let own_releasing = app_state.active_users.pending_release_allocations(req.username).await; + + let probe_tier = |index: usize| { + app_state.active_provider.capacity_probe( + &req.group.members[index].input_name, + req.priority, + req.kind, + req.lease.map(|l| l.provider_owner()), + &own_releasing, + ) + }; + + let mut remaining = ranked; + let mut probed_tiers: Vec<(usize, CapacityTier)> = + remaining.iter().map(|c| (c.index, probe_tier(c.index))).collect(); + + if let Some(hook) = after_probe { + hook(app_state); + } + + // Tier 1: Free ∪ OwnRelease + let tier1_match = |tier| match tier { + CapacityTier::OwnRelease => Some(true), + CapacityTier::Free => Some(false), + _ => None, + }; + if let Some(acquired) = select_tier_candidates( + app_state, + &req, + &mut remaining, + &mut probed_tiers, + all_circuits_open, + tier1_match, + &probe_tier, + ) { + return Ok(acquired); + } + + // Tier 2: Grace (only if allow_provider_grace) + if req.allow_provider_grace { + let tier2_match = |tier| match tier { + CapacityTier::Grace => Some(true), + CapacityTier::Free => Some(false), + _ => None, + }; + if let Some(acquired) = select_tier_candidates( + app_state, + &req, + &mut remaining, + &mut probed_tiers, + all_circuits_open, + tier2_match, + &probe_tier, + ) { + return Ok(acquired); + } + } + + // Tier 3: Preempt on the best-ranked candidate + if let Some(acquired) = try_acquire_preempt(app_state, &req, &remaining, all_circuits_open).await { + return Ok(acquired); + } + + Err(GroupSelectionFailure::Exhausted) +} + +pub(crate) async fn resolve_hls_group_entry( + app_state: &Arc, + target: &tuliprox_core::model::ConfigTarget, + group: Arc, + req: GroupSelectionRequest<'_>, +) -> Result { + let max_attempts = u32::from(req.selection.max_start_attempts.max(1)); + let mut attempts = 0; + let start_time = std::time::Instant::now(); + let budget = Duration::from_secs(req.selection.startup_budget_secs); + let mut exclude: Vec = req.exclude.to_vec(); + let demoted_inputs: Vec> = req.demoted_inputs.to_vec(); + + while attempts < max_attempts && (req.selection.startup_budget_secs == 0 || start_time.elapsed() < budget) { + attempts += 1; + let attempt_req = GroupSelectionRequest { + target_name: req.target_name, + group: req.group, + policy: req.policy, + mode: req.mode.clone(), + selection: req.selection, + addr: req.addr, + username: req.username, + priority: req.priority, + kind: req.kind, + lease: req.lease, + allow_provider_grace: req.allow_provider_grace, + exclude: &exclude, + demoted_inputs: &demoted_inputs, + prefer: if attempts == 1 { req.prefer } else { None }, + }; + + let acquired = acquire_group_member(app_state, attempt_req).await?; + let member = group.members[acquired.index].clone(); + let Some(member_input) = app_state.app_config.get_input_by_name(&member.input_name) else { + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_classified_failure_with( + guard, + tuliprox_session::channel_group::FailureClass::Stream, + std::time::Instant::now(), + ); + } else { + app_state.member_health.record_classified_failure( + &key, + tuliprox_session::channel_group::FailureClass::Stream, + std::time::Instant::now(), + ); + } + drop(acquired.handle); + exclude.push(acquired.index); + continue; + }; + + let mut stream_channel = if target.has_output(shared::model::TargetType::Xtream) { + tuliprox_repository::target_stream_item::member_to_xtream_item(&group, acquired.index, target) + .to_stream_channel(target.id) + } else { + tuliprox_repository::target_stream_item::member_to_m3u_item(&group, acquired.index, target) + .to_stream_channel(target.id) + }; + stream_channel.group_virtual_id = Some(group.group_vid.get()); + + let stream_url = Arc::clone(&member.url); + return Ok(ResolvedGroupPlayback { acquired, member, input: member_input, stream_url, stream_channel }); + } + + Err(GroupSelectionFailure::Exhausted) +} diff --git a/backend/app/src/api/api_utils/mod.rs b/backend/app/src/api/api_utils/mod.rs index d689cb798..4793a2bf3 100644 --- a/backend/app/src/api/api_utils/mod.rs +++ b/backend/app/src/api/api_utils/mod.rs @@ -233,11 +233,12 @@ pub enum ResourceFetchPolicy { const API_STREAM_CHUNK_SIZE: usize = 64 * 1024; #[cfg(test)] -mod tests; +pub(crate) mod tests; mod activation; mod auth; mod catchup; +pub(crate) mod channel_group; mod clock; mod local_files; mod maintenance; @@ -318,12 +319,13 @@ pub use self::{ strategy::redirect_response, streaming::{ force_provider_stream_response, is_hls_stream_share_enabled, is_seek_request, is_seekable_media_request, - is_stream_share_enabled, + is_stream_share_enabled, sticky_playback_context, }, }; pub(crate) use self::{ auth::{input_for_user, with_recording_headers}, catchup::get_catchup_session_ttl_secs, + channel_group::{resolve_hls_group_entry, GroupSelectionRequest, PlaybackSource}, local_files::local_stream_response, maintenance::{blocks_new_playback, is_running_playback, maintenance_rejection_response}, provider_acquire::{acquire_exact_provider_handle, stream_response_with_provider_handle}, diff --git a/backend/app/src/api/api_utils/provider_acquire.rs b/backend/app/src/api/api_utils/provider_acquire.rs index b86083b3f..9c0d4cce3 100644 --- a/backend/app/src/api/api_utils/provider_acquire.rs +++ b/backend/app/src/api/api_utils/provider_acquire.rs @@ -1,14 +1,18 @@ use super::{ activate_session_before_stream_open, admission_failure_response, blocks_new_playback, + channel_group::{ + acquire_group_member, group_shared_key, GroupSelectionFailure, GroupSelectionRequest, PlaybackSource, + }, cleanup_failed_detected_catchup_hls, connection_priority_for_kind, create_stream_response_details, detected_catchup_hls_response, get_session_reservation_ttl_secs, get_stream_options, is_socket_bound_playback_session, is_stream_share_enabled, mark_response_as_uncompressed, no_custom_video_fallback_status, prepare_body_stream, prepare_stream_metering, probe_catchup_payload, - record_connect_failed_attempt, reentry_suppressed_response, resolve_request_url_for_logging, - resolve_xtream_vod_provider_url, select_provider_stream_url, should_pin_provider_for_session, - stream_admission_rejected_response, try_shared_stream_response_if_any, try_unwrap_body, CatchupPayload, - ConnectFailedAttempt, DetectedCatchupHlsResponseParams, ExactProviderAcquire, PlaybackRequestClass, - SessionActivationRequest, StreamResponseMode, StreamingAcquireOptions, LEASE_EXPIRY_RECHECK_FLOOR, + record_connect_failed_attempt, reentry_suppressed_response, resolve_initial_stalker_playback_url, + resolve_request_url_for_logging, resolve_xtream_vod_provider_url, select_provider_stream_url, + should_pin_provider_for_session, stream_admission_rejected_response, try_shared_stream_response_if_any, + try_unwrap_body, CatchupPayload, ConnectFailedAttempt, DetectedCatchupHlsResponseParams, ExactProviderAcquire, + PlaybackRequestClass, SessionActivationRequest, StreamResponseMode, StreamingAcquireOptions, + LEASE_EXPIRY_RECHECK_FLOOR, }; use crate::{ api::model::{ @@ -25,13 +29,21 @@ use axum::{ http::{HeaderMap, StatusCode}, response::IntoResponse, }; +use futures::StreamExt; use log::{debug, error, log_enabled, trace}; use shared::{ - model::{ConnectFailureReason, FailureStage, PlaylistItemType, StreamChannel, UserConnectionPermission, VirtualId}, + model::{ + ConnectFailureReason, EventMessage, FailureStage, PlaylistItemType, StreamChannel, UserConnectionPermission, + VirtualId, + }, utils::{extract_extension_from_url, sanitize_sensitive_info, Internable}, }; use std::{borrow::Cow, sync::Arc, time::Duration}; -use tuliprox_session::ProviderSessionHeaders; +use tuliprox_session::{ + channel_group::{member_key, FailureClass}, + stream::StreamDetails, + ProviderSessionHeaders, +}; pub(super) fn create_unmapped_provider_stream(app_config: &AppConfig) -> ProviderStreamState { ProviderStreamState::Custom { @@ -293,10 +305,9 @@ pub(crate) async fn stream_response_with_provider_handle( session_token: &str, request_class: Option, mut stream_channel: StreamChannel, - stream_url: &str, + source: PlaybackSource<'_>, pinned_provider: Option<&Arc>, req_headers: &HeaderMap, - input: &Arc, target: &Arc, user: &ProxyUserCredentials, connection_permission: UserConnectionPermission, @@ -305,26 +316,45 @@ pub(crate) async fn stream_response_with_provider_handle( grace_mode: Option, preacquired_provider_handle: Option, ) -> impl IntoResponse + Send { - if blocks_new_playback(app_state, input, user, preacquired_provider_handle.is_some()) { - return admission_failure_response( - app_state, - fingerprint, - user, - stream_channel, - Arc::clone(&input.name), - req_headers, - ConnectFailureReason::Maintenance, - ); + if let PlaybackSource::Single { input, .. } = &source { + if blocks_new_playback(app_state, input, user, preacquired_provider_handle.is_some()) { + return admission_failure_response( + app_state, + fingerprint, + user, + stream_channel, + Arc::clone(&input.name), + req_headers, + ConnectFailureReason::Maintenance, + ); + } } let _transition_guard = app_state.active_users.acquire_playback_transition(&user.username, session_token).await; - let request_log_stream_url = resolve_request_url_for_logging(input, stream_url); + + if let PlaybackSource::Group { group, .. } = &source { + stream_channel.group_virtual_id = Some(group.group_vid.get()); + } + + let (initial_input, initial_stream_url) = match &source { + PlaybackSource::Single { input, stream_url } => (Arc::clone(input), (*stream_url).to_string()), + PlaybackSource::Group { group, .. } => { + let first_member = &group.members[0]; + let cand_input = app_state + .app_config + .get_input_by_name(&first_member.input_name) + .unwrap_or_else(|| Arc::new(ConfigInput::default())); + (cand_input, first_member.url.to_string()) + } + }; + + let request_log_stream_url = resolve_request_url_for_logging(&initial_input, &initial_stream_url); if log_enabled!(log::Level::Trace) { trace!("Try to open stream {}", sanitize_sensitive_info(request_log_stream_url.as_ref())); } - let virtual_id = stream_channel.virtual_id; + let virtual_id = stream_channel.client_virtual_id(); let item_type = stream_channel.item_type; - let playback_extension = extract_extension_from_url(stream_url); + let playback_extension = extract_extension_from_url(&initial_stream_url); let socket_bound = is_socket_bound_playback_session(item_type, playback_extension); let mut connection_permission = connection_permission; let mut connection_kind = connection_kind; @@ -332,13 +362,13 @@ pub(crate) async fn stream_response_with_provider_handle( app_state, SessionActivationRequest { fingerprint, - input, + input: &initial_input, user, session_token, request_class, virtual_id: VirtualId::new(virtual_id), item_type, - stream_url, + stream_url: &initial_stream_url, connection_permission, connection_kind, granted_grace_mode: grace_mode, @@ -353,14 +383,19 @@ pub(crate) async fn stream_response_with_provider_handle( let allow_shared_reuse = connection_permission != UserConnectionPermission::Exhausted || allow_exhausted_shared_reconnect; + let shared_lookup_key: Arc = match &source { + PlaybackSource::Group { group, .. } => group_shared_key(target.id, group.group_vid.get()), + PlaybackSource::Single { stream_url, .. } => Arc::from(*stream_url), + }; + let share_stream = is_stream_share_enabled(item_type, target); let _shared_lock = if share_stream { - let write_lock = app_state.app_config.file_locks.write_lock_str(stream_url).await; + let write_lock = app_state.app_config.file_locks.write_lock_str(shared_lookup_key.as_ref()).await; if allow_shared_reuse { if let Some(value) = try_shared_stream_response_if_any( app_state, - stream_url, + shared_lookup_key.as_ref(), fingerprint, user, connection_permission, @@ -382,7 +417,7 @@ pub(crate) async fn stream_response_with_provider_handle( if item_type == PlaylistItemType::Live && allow_shared_reuse { if let Some(value) = try_shared_stream_response_if_any( app_state, - stream_url, + shared_lookup_key.as_ref(), fingerprint, user, connection_permission, @@ -394,7 +429,10 @@ pub(crate) async fn stream_response_with_provider_handle( ) .await { - debug_if_enabled!("Opportunistic shared stream reuse for {}", sanitize_sensitive_info(stream_url)); + debug_if_enabled!( + "Opportunistic shared stream reuse for {}", + sanitize_sensitive_info(shared_lookup_key.as_ref()) + ); return value.into_response(); } } @@ -419,7 +457,7 @@ pub(crate) async fn stream_response_with_provider_handle( fingerprint, user, stream_channel: stream_channel.clone(), - provider_name: input.name.clone(), + provider_name: initial_input.name.clone(), req_headers, reason: ConnectFailureReason::UserConnectionsExhausted, failure_stage: FailureStage::Admission, @@ -432,53 +470,61 @@ pub(crate) async fn stream_response_with_provider_handle( .into_response(); } - let stream_options = get_stream_options(&app_state.app_config, StreamResponseMode::Stream); - let session_state = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - let pinned_provider = pinned_provider.filter(|provider| { - item_type.is_live() - || session_state - .as_ref() - .is_some_and(|session| session.media_started.load(std::sync::atomic::Ordering::Acquire)) - || app_state.active_provider.should_reuse_playback_provider(session_token, provider) - }); - let provider_session_headers = - session_state.as_ref().and_then(|session| session.provider_session_headers_for(stream_url)); - let mut stream_details = match create_stream_response_details( - app_state, - &stream_options, - stream_url, - &user.username, - fingerprint, - req_headers, - input, - &stream_channel, - item_type, - if item_type == PlaylistItemType::Catchup { - crate::api::model::ProviderContentRepresentationMode::Identity - } else { - crate::api::model::ProviderContentRepresentationMode::PreserveOrigin - }, - share_stream, - connection_permission, - pinned_provider, - pinned_provider.is_none() || allows_provider_pool_failover(item_type), - true, - VirtualId::new(stream_channel.virtual_id), - connection_priority_for_kind(user, connection_kind), - connection_kind, - false, - Some(session_token), - provider_session_headers.as_deref(), - pinned_provider.is_some(), - grace_mode.map(|m| matches!(m, crate::api::model::GraceMode::Hold)), - activation.grace_context.clone(), - None, - preacquired_provider_handle, - ) - .await + if matches!(&source, PlaybackSource::Group { .. }) + && connection_permission == UserConnectionPermission::GracePeriod + && matches!(grace_mode, Some(crate::api::model::GraceMode::Hold)) { - Ok(stream_details) => stream_details, - Err(err) => { + let grace_period = app_state.get_grace_options(); + let hold_duration = Duration::from_millis(grace_period.period_millis); + let deadline = tokio::time::Instant::now() + hold_duration; + let mut passed = false; + loop { + if app_state.active_users.user_connections(&user.username).await < user.max_connections { + passed = true; + break; + } + if tokio::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + if !passed { + if let Some(ref ctx) = activation.grace_context { + let eviction_guard = if socket_bound { + tuliprox_session::admission::EvictionReentryGuard::SocketPlayback { + virtual_id: shared::model::VirtualId(virtual_id), + } + } else { + tuliprox_session::admission::EvictionReentryGuard::Session(session_token) + }; + let remaining_result = tuliprox_session::admission::evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + tuliprox_session::admission::AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: true, + session_token: Some(session_token), + activate_unbound_session: true, + eviction_reentry_guard: eviction_guard, + }, + ctx, + activation.admission.kind(), + ) + .await; + if matches!( + remaining_result.admission.permission(), + UserConnectionPermission::Allowed | UserConnectionPermission::GracePeriod + ) { + passed = true; + } + } + } + + if !passed { app_state .active_users .release_unbound_session_reservation( @@ -488,8 +534,457 @@ pub(crate) async fn stream_response_with_provider_handle( activation.placeholder_transition_version.is_some(), ) .await; - error!("Failed to stream: {err}"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel: stream_channel.clone(), + provider_name: initial_input.name.clone(), + req_headers, + reason: ConnectFailureReason::UserConnectionsExhausted, + failure_stage: FailureStage::Admission, + }); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) + .into_response(); + } + connection_permission = UserConnectionPermission::Allowed; + } + + let stream_options = get_stream_options(&app_state.app_config, StreamResponseMode::Stream); + + let (mut stream_details, input, stream_url, mut stream_channel) = match source { + PlaybackSource::Single { input, stream_url } => { + let session_state = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; + let pinned_provider = pinned_provider.filter(|provider| { + item_type.is_live() + || session_state + .as_ref() + .is_some_and(|session| session.media_started.load(std::sync::atomic::Ordering::Acquire)) + || app_state.active_provider.should_reuse_playback_provider(session_token, provider) + }); + let provider_session_headers = + session_state.as_ref().and_then(|session| session.provider_session_headers_for(stream_url)); + let details = match create_stream_response_details( + app_state, + &stream_options, + stream_url, + &user.username, + fingerprint, + req_headers, + input, + &stream_channel, + item_type, + if item_type == PlaylistItemType::Catchup { + crate::api::model::ProviderContentRepresentationMode::Identity + } else { + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin + }, + share_stream, + connection_permission, + pinned_provider, + pinned_provider.is_none() || allows_provider_pool_failover(item_type), + true, + VirtualId::new(stream_channel.virtual_id), + connection_priority_for_kind(user, connection_kind), + connection_kind, + false, + Some(session_token), + provider_session_headers.as_deref(), + pinned_provider.is_some(), + grace_mode.map(|m| matches!(m, crate::api::model::GraceMode::Hold)), + activation.grace_context.clone(), + None, + preacquired_provider_handle, + ) + .await + { + Ok(stream_details) => stream_details, + Err(err) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + error!("Failed to stream: {err}"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + (details, Arc::clone(input), Arc::::from(stream_url), stream_channel) + } + PlaybackSource::Group { group, mode, prefer } => { + let (selection_cfg, allow_provider_grace) = + target.options.as_ref().and_then(|o| o.channel_groups.as_ref()).map_or_else( + || (shared::model::ChannelGroupSelectionConfig::default(), true), + |cg| (cg.selection, cg.allow_provider_grace), + ); + let policy = group.policy; + + let target_name: Arc = Arc::from(target.name.as_str()); + let mut exclude: Vec = Vec::new(); + let mut demoted_inputs: Vec> = Vec::new(); + let mut attempts: u32 = 0; + let max_attempts = u32::from(selection_cfg.max_start_attempts.max(1)); + let start_time = std::time::Instant::now(); + let budget = Duration::from_secs(selection_cfg.startup_budget_secs); + + let mut final_details: Option<(StreamDetails, Arc, Arc, StreamChannel)> = None; + + while attempts < max_attempts && (selection_cfg.startup_budget_secs == 0 || start_time.elapsed() < budget) { + attempts += 1; + + let req = GroupSelectionRequest { + target_name: &target_name, + group: &group, + policy, + mode: mode.clone(), + selection: selection_cfg, + addr: &fingerprint.addr, + username: &user.username, + priority: connection_priority_for_kind(user, connection_kind), + kind: connection_kind, + lease: None, + allow_provider_grace, + exclude: &exclude, + demoted_inputs: &demoted_inputs, + prefer: if attempts == 1 { prefer } else { None }, + }; + + let acquired = match acquire_group_member(app_state, req).await { + Ok(acq) => acq, + Err(GroupSelectionFailure::Exhausted | GroupSelectionFailure::NoCandidate) => { + break; + } + }; + + let member = group.members[acquired.index].clone(); + let Some(member_input) = app_state.app_config.get_input_by_name(&member.input_name) else { + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_classified_failure_with( + guard, + FailureClass::Stream, + std::time::Instant::now(), + ); + } else { + app_state.member_health.record_classified_failure( + &key, + FailureClass::Stream, + std::time::Instant::now(), + ); + } + drop(acquired.handle); + exclude.push(acquired.index); + continue; + }; + + let mut member_channel = stream_channel.clone(); + member_channel.virtual_id = member.member_vid.get(); + member_channel.group_virtual_id = Some(group.group_vid.get()); + member_channel.input_name = Arc::clone(&member.input_name); + member_channel.item_type = member.item_type; + member_channel.cluster = member.cluster; + member_channel.title = Arc::clone(&member.title); + member_channel.url = Arc::clone(&member.url); + member_channel.provider_id = member.provider_id; + + let member_url = if member_input.input_type.is_stalker() { + let Ok(url) = resolve_initial_stalker_playback_url( + app_state, + &member_input, + user, + member.provider_id, + member.cluster, + member.item_type, + &member.url, + ) + .await + else { + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_classified_failure_with( + guard, + FailureClass::Stream, + std::time::Instant::now(), + ); + } else { + app_state.member_health.record_classified_failure( + &key, + FailureClass::Stream, + std::time::Instant::now(), + ); + } + drop(acquired.handle); + exclude.push(acquired.index); + continue; + }; + url + } else { + Arc::clone(&member.url) + }; + + let details_result = create_stream_response_details( + app_state, + &stream_options, + member_url.as_ref(), + &user.username, + fingerprint, + req_headers, + &member_input, + &member_channel, + member.item_type, + if member.item_type == PlaylistItemType::Catchup { + crate::api::model::ProviderContentRepresentationMode::Identity + } else { + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin + }, + share_stream, + connection_permission, + None, + true, + true, + VirtualId::new(member.member_vid.get()), + connection_priority_for_kind(user, connection_kind), + connection_kind, + false, + Some(session_token), + None, + true, + None, + None, + None, + Some(acquired.handle), + ) + .await; + + let Ok(mut details) = details_result else { + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_classified_failure_with( + guard, + FailureClass::Stream, + std::time::Instant::now(), + ); + } else { + app_state.member_health.record_classified_failure( + &key, + FailureClass::Stream, + std::time::Instant::now(), + ); + } + exclude.push(acquired.index); + continue; + }; + + if let Some(mut provider_stream) = details.stream.take() { + if let Some((_, _, _, Some(_custom_type))) = details.stream_info { + let failure_class = + if details.custom_reason == Some(ProviderStreamCustomReason::ProviderExhausted) { + FailureClass::Capacity + } else { + FailureClass::Stream + }; + if matches!(failure_class, FailureClass::Auth | FailureClass::Capacity) { + demoted_inputs.push(Arc::clone(&member.input_name)); + } + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_classified_failure_with( + guard, + failure_class, + std::time::Instant::now(), + ); + } else { + app_state.member_health.record_classified_failure( + &key, + failure_class, + std::time::Instant::now(), + ); + } + app_state.connection_manager.release_managed_provider_handle(details.provider_handle); + exclude.push(acquired.index); + continue; + } + + let is_adaptive_or_playlist = { + if member.item_type.is_live_adaptive() { + true + } else if let Some((headers, _, _, _)) = &details.stream_info { + headers.iter().any(|(name, val)| { + name.as_str().eq_ignore_ascii_case("content-type") && { + let lower = val.to_ascii_lowercase(); + lower.contains("mpegurl") || lower.contains("dash+xml") + } + }) + } else { + false + } + }; + + let mut sync_ok = true; + let mut buffered_chunks = Vec::new(); + + if !is_adaptive_or_playlist { + let mut total_bytes = 0; + let mut read_failed = false; + let first_byte_timeout = Duration::from_secs(3); + + while total_bytes < 564 { + if let Ok(Some(Ok(chunk))) = + tokio::time::timeout(first_byte_timeout, provider_stream.next()).await + { + if chunk.is_empty() { + continue; + } + total_bytes += chunk.len(); + buffered_chunks.push(chunk); + } else { + read_failed = true; + break; + } + } + + sync_ok = if read_failed && buffered_chunks.is_empty() { + false + } else { + let mut flat = Vec::with_capacity(total_bytes.min(564)); + for c in &buffered_chunks { + let need = 564_usize.saturating_sub(flat.len()); + flat.extend_from_slice(&c[..c.len().min(need)]); + if flat.len() >= 564 { + break; + } + } + tuliprox_mpegts::ts_sync_offset(&flat).is_some() + }; + } + + if !sync_ok { + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_classified_failure_with( + guard, + FailureClass::Stream, + std::time::Instant::now(), + ); + } else { + app_state.member_health.record_classified_failure( + &key, + FailureClass::Stream, + std::time::Instant::now(), + ); + } + app_state.connection_manager.release_managed_provider_handle(details.provider_handle); + exclude.push(acquired.index); + continue; + } + + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_success_with(guard); + } else { + app_state.member_health.record_success(&key); + } + + app_state.event_manager.send_event(EventMessage::ChannelGroupMemberSelected { + target_id: target.id, + group_vid: group.group_vid.get(), + member_vid: member.member_vid.get(), + tier: acquired.tier, + }); + + app_state + .active_users + .bind_member( + &user.username, + session_token, + tuliprox_session::MemberBinding { + group_vid: group.group_vid.get(), + member_vid: member.member_vid.get(), + member_index: acquired.index, + input_name: Arc::clone(&member.input_name), + }, + ) + .await; + + if buffered_chunks.is_empty() { + details.stream = Some(provider_stream); + } else { + let prepended = + futures::stream::iter(buffered_chunks.into_iter().map(Ok)).chain(provider_stream); + details.stream = Some(prepended.boxed()); + } + + final_details = Some((details, member_input, member_url, member_channel)); + break; + } + + let failure_class = if let Some((_, status, _, _)) = &details.stream_info { + FailureClass::from_status(*status) + } else if details.custom_reason == Some(ProviderStreamCustomReason::ProviderExhausted) { + FailureClass::Capacity + } else { + FailureClass::Stream + }; + + if matches!(failure_class, FailureClass::Auth | FailureClass::Capacity) { + demoted_inputs.push(Arc::clone(&member.input_name)); + } + + let key = member_key(&member); + if let Some(guard) = acquired.half_open { + app_state.member_health.record_classified_failure_with( + guard, + failure_class, + std::time::Instant::now(), + ); + } else { + app_state.member_health.record_classified_failure(&key, failure_class, std::time::Instant::now()); + } + + app_state.connection_manager.release_managed_provider_handle(details.provider_handle); + exclude.push(acquired.index); + } + + let Some((details, inp, url, ch)) = final_details else { + app_state.event_manager.send_event(EventMessage::ChannelGroupExhausted { + target_id: target.id, + group_vid: group.group_vid.get(), + }); + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel: stream_channel.clone(), + provider_name: initial_input.name.clone(), + req_headers, + reason: ConnectFailureReason::ProviderConnectionsExhausted, + failure_stage: FailureStage::ProviderOpen, + }); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::ProviderConnectionsExhausted, + ) + .into_response(); + }; + (details, inp, url, ch) } }; @@ -505,13 +1000,13 @@ pub(crate) async fn stream_response_with_provider_handle( manifest, user, target, - input, + input: &input, fingerprint, session_token, virtual_id: VirtualId::new(virtual_id), connection_permission, connection_kind, - fallback_stream_url: stream_url, + fallback_stream_url: &stream_url, }) .await; } @@ -539,8 +1034,9 @@ pub(crate) async fn stream_response_with_provider_handle( .as_ref() .map(|(h, sc, response_url, cvt)| (h.clone(), *sc, response_url.clone(), *cvt)); let provider_name = stream_details.provider_name.clone(); - let actual_request_url = stream_details.request_url.clone().unwrap_or_else(|| Arc::::from(stream_url)); - let log_actual_request_url = resolve_request_url_for_logging(input, actual_request_url.as_ref()); + let actual_request_url = + stream_details.request_url.clone().unwrap_or_else(|| Arc::::from(stream_url.as_ref())); + let log_actual_request_url = resolve_request_url_for_logging(&input, actual_request_url.as_ref()); debug_if_enabled!( "Provider request mapping: allocated_provider={} actual_request_url={}", @@ -603,7 +1099,7 @@ pub(crate) async fn stream_response_with_provider_handle( }; let mut metering = prepare_stream_metering( app_state, - stream_url, + if is_stream_shared { shared_lookup_key.as_ref() } else { stream_url.as_ref() }, is_stream_shared, stream_details.stream.is_some(), stream_details.has_deferred_provider_open(), @@ -722,7 +1218,7 @@ pub(crate) async fn stream_response_with_provider_handle( active_provider: &app_state.active_provider, connection_manager: &app_state.connection_manager, }, - stream_url, + shared_lookup_key.as_ref(), stream, &fingerprint.addr, shared_subscriber_id, @@ -774,7 +1270,7 @@ pub(crate) async fn stream_response_with_provider_handle( .and_then(|(_, _, u, _)| u.as_ref()) .map_or_else(|| Cow::Owned(actual_request_url.to_string()), |url| Cow::Owned(url.to_string())) }; - let log_session_url = resolve_request_url_for_logging(input, session_url.as_ref()); + let log_session_url = resolve_request_url_for_logging(&input, session_url.as_ref()); if log_enabled!(log::Level::Debug) { if log_session_url.eq(log_actual_request_url.as_ref()) { debug!( diff --git a/backend/app/src/api/api_utils/streaming.rs b/backend/app/src/api/api_utils/streaming.rs index cea64b0d5..406c48f92 100644 --- a/backend/app/src/api/api_utils/streaming.rs +++ b/backend/app/src/api/api_utils/streaming.rs @@ -7,8 +7,9 @@ use super::{ resolve_streaming_strategy_with_provider_handle, session_reacquire_cleanup_addrs, should_defer_provider_open_for_grace_hold, should_refresh_stalker_playback, split_provider_stream_open, stalker_stream_kind, stream_admission_rejected_response, stream_response_with_provider_handle, try_unwrap_body, - ConnectFailedAttempt, CurrentSessionGuard, ForceStreamRequestContext, PlaybackRequestClass, RedirectParams, - ResourceFetchPolicy, StreamMeteringConfig, StreamOptions, StreamResponseMode, HLS_MEDIA_CAPACITY_WAIT, + ConnectFailedAttempt, CurrentSessionGuard, ForceStreamRequestContext, PlaybackRequestClass, PlaybackSource, + RedirectParams, ResourceFetchPolicy, StreamMeteringConfig, StreamOptions, StreamResponseMode, + HLS_MEDIA_CAPACITY_WAIT, }; use crate::{ api::{ @@ -608,6 +609,24 @@ pub(super) fn no_custom_video_fallback_status(app_config: &AppConfig) -> StatusC } } +pub fn sticky_playback_context( + app_state: &Arc, + session: &UserSession, + fallback_input: &Arc, + fallback_channel: StreamChannel, +) -> (Arc, StreamChannel) { + if let Some(binding) = &session.member_binding { + if let Some(member_input) = app_state.app_config.get_input_by_name(&binding.input_name) { + let mut channel = fallback_channel; + channel.virtual_id = binding.member_vid; + channel.group_virtual_id = Some(binding.group_vid); + channel.input_name = Arc::clone(&binding.input_name); + return (member_input, channel); + } + } + (Arc::clone(fallback_input), fallback_channel) +} + pub async fn force_provider_stream_response( fingerprint: &Fingerprint, app_state: &Arc, @@ -657,11 +676,13 @@ pub(super) async fn force_stream_response( fingerprint: &Fingerprint, app_state: &Arc, user_session: &UserSession, - mut stream_channel: StreamChannel, + stream_channel: StreamChannel, ctx: ForceStreamRequestContext<'_>, grace_mode: Option, response_mode: StreamResponseMode, ) -> axum::response::Response { + let (sticky_input, mut stream_channel) = + sticky_playback_context(app_state, user_session, ctx.input, stream_channel); let hls_resource = response_mode == StreamResponseMode::HlsResource; // Finite HLS resources stay pinned to the session account and never rebind or clean up // other sockets, so they skip the session transition gate. Holding it across the @@ -739,7 +760,7 @@ pub(super) async fn force_stream_response( &ctx.user.username, fingerprint, ctx.req_headers, - ctx.input, + &sticky_input, &stream_channel, item_type, ctx.content_representation, @@ -952,7 +973,7 @@ pub(super) async fn force_stream_response( debug_if_enabled!( "Streaming provider forced stream request from {}", sanitize_sensitive_info( - resolve_request_url_for_logging(ctx.input, user_session.stream_url.as_ref()).as_ref() + resolve_request_url_for_logging(&sticky_input, user_session.stream_url.as_ref()).as_ref() ) ); let mut response = try_unwrap_body!(response.body(body_stream)); @@ -990,10 +1011,9 @@ pub(crate) async fn stream_response( session_token: &str, request_class: Option, stream_channel: StreamChannel, - stream_url: &str, + source: PlaybackSource<'_>, pinned_provider: Option<&Arc>, req_headers: &HeaderMap, - input: &Arc, target: &Arc, user: &ProxyUserCredentials, connection_permission: UserConnectionPermission, @@ -1007,10 +1027,9 @@ pub(crate) async fn stream_response( session_token, request_class, stream_channel, - stream_url, + source, pinned_provider, req_headers, - input, target, user, connection_permission, diff --git a/backend/app/src/api/api_utils/tests.rs b/backend/app/src/api/api_utils/tests.rs index 77a58cc05..c26639329 100644 --- a/backend/app/src/api/api_utils/tests.rs +++ b/backend/app/src/api/api_utils/tests.rs @@ -37,3 +37,8 @@ mod stream_responses_provider_capacity; mod stream_responses_provider_fallback; mod stream_responses_response_headers; mod stream_responses_session_activation; + +mod channel_group_playback; +mod channel_group_selection; +mod channel_group_sticky; +pub(crate) mod group_playback_support; diff --git a/backend/app/src/api/api_utils/tests/admission_session_admission.rs b/backend/app/src/api/api_utils/tests/admission_session_admission.rs index cd02fedfe..2df1d54e9 100644 --- a/backend/app/src/api/api_utils/tests/admission_session_admission.rs +++ b/backend/app/src/api/api_utils/tests/admission_session_admission.rs @@ -4,7 +4,7 @@ use super::{ create_test_provider_app_config, create_test_shared_target, load_test_channel, load_test_user, resolve_admission_with_strategies, should_allow_exhausted_shared_reconnect, should_defer_provider_open_for_grace_hold, spawn_load_test_origin, stream_admission_rejected_response, - stream_response, AdmissionRequest, EvictionReentryGuard, SessionActivationRequest, + stream_response, AdmissionRequest, EvictionReentryGuard, PlaybackSource, SessionActivationRequest, }; use crate::{ api::model::{CustomVideoStreamType, UserSession}, @@ -99,10 +99,9 @@ async fn shared_admission_rejection_does_not_leak_meter_registration() { "shared-session-adm", None, channel, - &stream_url, + PlaybackSource::Single { input: &input, stream_url: &stream_url }, None, &HeaderMap::default(), - &input, &target, &user, UserConnectionPermission::Allowed, diff --git a/backend/app/src/api/api_utils/tests/channel_group_playback.rs b/backend/app/src/api/api_utils/tests/channel_group_playback.rs new file mode 100644 index 000000000..6db03c526 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/channel_group_playback.rs @@ -0,0 +1,127 @@ +use super::group_playback_support::{ts_fixture, GroupPlaybackEnv}; +use shared::model::AdmissionStrategy; +use tuliprox_session::channel_group::CircuitView; + +#[tokio::test] +async fn xtream_group_playback_uses_member_input() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]) + .with_qos("korath", 2, 95) + .with_qos("solis", 1, 20) + .build() + .await; + let response = env.get_xtream_live(env.group_vid).await; + assert_eq!(response.status(), 200); + assert_eq!(env.origin_hits("korath"), 1); + assert_eq!(env.origin_hits("solis"), 0); +} + +#[tokio::test] +async fn start_failover_moves_to_next_member_on_open_failure() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).origin_status("solis", 503).build().await; + let response = env.get_xtream_live(env.group_vid).await; + assert_eq!(response.status(), 200); + assert_eq!(env.origin_hits("korath"), 1); + assert!(matches!(env.circuit("solis"), CircuitView::Counting(1))); +} + +#[tokio::test] +async fn capacity_rejection_moves_to_other_input_without_circuit() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).origin_status("solis", 429).build().await; + let response = env.get_xtream_live(env.group_vid).await; + assert_eq!(response.status(), 200); + assert_eq!(env.origin_hits("korath"), 1); + assert!(matches!(env.circuit("solis"), CircuitView::Closed)); +} + +#[tokio::test] +async fn non_ts_first_chunk_fails_over() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]) + .origin_body("solis", "video/mp2t", b"Channel offline, please try later".to_vec()) + .build() + .await; + assert_eq!(env.get_xtream_live(env.group_vid).await.status(), 200); + assert_eq!(env.origin_hits("korath"), 1); + assert!(matches!(env.circuit("solis"), CircuitView::Counting(1))); +} + +#[tokio::test] +async fn ts_with_leading_garbage_is_accepted() { + let mut body = vec![0u8; 17]; + body.extend(ts_fixture()); + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).origin_body("solis", "video/mp2t", body).build().await; + assert_eq!(env.get_xtream_live(env.group_vid).await.status(), 200); + assert_eq!(env.origin_hits("korath"), 0); +} + +#[tokio::test] +async fn auth_failure_counts_for_member_and_input() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).origin_status("solis", 401).build().await; + assert_eq!(env.get_xtream_live(env.group_vid).await.status(), 200); + assert_eq!(env.origin_hits("korath"), 1); + assert!(matches!(env.circuit("solis"), CircuitView::Counting(1))); + assert!(matches!(env.input_circuit("solis"), CircuitView::Closed)); // one member only, input stays closed +} + +#[tokio::test] +async fn member_vid_request_plays_group_preferring_that_member() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).with_qos("korath", 2, 95).build().await; + let response = env.get_xtream_live(env.member_vid("solis")).await; + assert_eq!(response.status(), 200); + assert_eq!((env.origin_hits("solis"), env.origin_hits("korath")), (1, 0)); // preferred despite lower QoS +} + +#[tokio::test] +async fn favourite_copy_plays_as_group() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).with_favourite().with_qos("korath", 2, 95).build().await; + assert_eq!(env.get_xtream_live(env.favourite_vid()).await.status(), 200); + assert_eq!(env.origin_hits("korath"), 1); +} + +#[tokio::test] +async fn non_group_playback_is_unchanged() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).build().await; + assert_eq!(env.get_xtream_live(env.plain_vid).await.status(), 200); + assert_eq!(env.origin_hits("solis"), 1); +} + +#[tokio::test] +async fn m3u_group_playback_uses_member_input() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).with_qos("korath", 2, 95).build().await; + assert_eq!(env.get_m3u_live(env.group_vid).await.status(), 200); + assert_eq!(env.origin_hits("solis"), 0); +} + +#[tokio::test] +async fn stalker_member_url_resolved_after_selection() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).stalker("solis").with_qos("korath", 2, 95).build().await; + assert_eq!(env.get_xtream_live(env.group_vid).await.status(), 200); + assert_eq!(env.origin_hits("solis"), 0); // no Stalker create_link against the representative +} + +#[tokio::test] +async fn admission_uses_group_vid_for_eviction() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]) + .user_limit(1) + .admission(vec![AdmissionStrategy::EvictUserSameIpOldest]) + .build() + .await; + let first = env.get_xtream_live(env.group_vid).await; + assert_eq!(first.status(), 200); + let second = env.get_xtream_live(env.plain_vid).await; + assert_eq!(second.status(), 200); + assert_eq!(env.active_session_vids("viewer").await, vec![env.plain_vid]); // group session evicted by its group vid +} + +#[tokio::test] +async fn grace_hold_selects_member_only_after_hold_passes() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]) + .user_limit(1) + .admission(vec![AdmissionStrategy::GraceHoldStream]) + .build() + .await; + let _first = env.get_xtream_live(env.plain_vid).await; + let slots_before = env.slots_in_use(); + let rejected = env.get_xtream_live(env.group_vid).await; // first stream still active → hold fails + assert_ne!(rejected.status(), 200); + assert_eq!(env.slots_in_use(), slots_before); // no member slot was taken during the hold +} diff --git a/backend/app/src/api/api_utils/tests/channel_group_selection.rs b/backend/app/src/api/api_utils/tests/channel_group_selection.rs new file mode 100644 index 000000000..bcf65f714 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/channel_group_selection.rs @@ -0,0 +1,234 @@ +use super::{ + super::channel_group::{ + acquire_group_member, acquire_group_member_with_race, GroupSelectionFailure, GroupSelectionRequest, + }, + support::{create_test_app_config, create_test_app_state_for_config}, +}; +use crate::api::model::AppState; +use arc_swap::ArcSwap; +use shared::{ + model::{ + ChannelGroupPolicy, ChannelGroupsConfig, InputType, PlaylistItemType, ReverseProxyConfigDto, StreamConfigDto, + VirtualId, XtreamCluster, + }, + utils::Internable, +}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_core::model::{Config, ConfigInput, ProviderHandle, ReverseProxyConfig, SourcesConfig}; +use tuliprox_repository::{ + channel_group_repository::{ChannelGroupRecord, GroupMember}, + ChannelGroupOrigin, +}; +use tuliprox_session::{ + active_provider_manager::{CapacityTier, ConnectionKind}, + channel_group::SelectionMode, +}; + +static CLIENT_ADDR: std::sync::LazyLock = + std::sync::LazyLock::new(|| "127.0.0.1:40000".parse().expect("valid client addr")); + +fn addr(port: u16) -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], port)) } + +fn xtream_input(id: u16, name: &str, max_connections: u16) -> Arc { + Arc::new(ConfigInput { + id, + name: name.intern(), + input_type: InputType::Xtream, + url: format!("http://{name}.invalid"), + username: Some("user".into()), + password: Some("pass".into()), + enabled: true, + max_connections, + ..ConfigInput::default() + }) +} + +fn group_member(input: &str, stream_id: &str, provider_id: u32, member_vid: u32) -> GroupMember { + GroupMember { + member_vid: VirtualId::new(member_vid), + input_name: input.into(), + provider_id, + input_stream_id: stream_id.into(), + url: format!("http://{input}.invalid/{stream_id}.ts").into(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + title: format!("{input} stream").into(), + archive: None, + epg_channel_id: None, + additional_properties: None, + } +} + +fn app_config_with_sources(sources: SourcesConfig) -> tuliprox_core::model::AppConfig { + let mut config = Config::default(); + let dto = ReverseProxyConfigDto { + stream: Some(StreamConfigDto { grace_period_millis: 2_000, ..StreamConfigDto::default() }), + ..ReverseProxyConfigDto::default() + }; + config.reverse_proxy = Some(ReverseProxyConfig::from(&dto)); + let mut app_config = create_test_app_config(); + app_config.config = Arc::new(ArcSwap::from_pointee(config)); + app_config.sources = Arc::new(ArcSwap::from_pointee(sources)); + app_config +} + +struct SelectionEnv { + app_state: Arc, + group: ChannelGroupRecord, + config: ChannelGroupsConfig, + target: Arc, +} + +impl SelectionEnv { + fn new(members: &[(&str, u16)]) -> Self { + let sources = SourcesConfig { + inputs: members + .iter() + .enumerate() + .map(|(i, (name, max))| xtream_input(u16::try_from(i + 1).expect("id"), name, *max)) + .collect(), + ..SourcesConfig::default() + }; + let app_state = create_test_app_state_for_config(Arc::new(app_config_with_sources(sources))); + let group = ChannelGroupRecord { + group_vid: VirtualId::new(1), + key: "zdf".into(), + origin: ChannelGroupOrigin::Auto, + policy: ChannelGroupPolicy::Priority, + orphaned: Vec::new(), + members: members + .iter() + .enumerate() + .map(|(i, (name, _))| { + group_member( + name, + &(i + 1).to_string(), + u32::try_from(i + 1).expect("provider_id"), + 100 + u32::try_from(i).expect("member_vid"), + ) + }) + .collect(), + }; + Self { + app_state, + group, + config: serde_json::from_str(r#"{"enabled":true}"#).expect("config"), + target: "Main".into(), + } + } + + fn hold(&self, input: &str, priority: i8) -> ProviderHandle { + self.app_state + .active_provider + .acquire_connection(&input.into(), &addr(200), priority, ConnectionKind::Normal) + .expect("hold slot") + } + + fn request<'a>(&'a self, exclude: &'a [usize], allow_grace: bool) -> GroupSelectionRequest<'a> { + GroupSelectionRequest { + target_name: &self.target, + group: &self.group, + policy: self.group.policy, + mode: SelectionMode::Live, + selection: self.config.selection, + addr: &CLIENT_ADDR, + username: "viewer", + priority: 0, + kind: ConnectionKind::Normal, + lease: None, + allow_provider_grace: allow_grace, + exclude, + demoted_inputs: &[], + prefer: None, + } + } + + fn slots_in_use(&self) -> usize { + self.app_state.active_provider.active_connections().unwrap_or_default().values().sum() + } +} + +#[tokio::test] +async fn free_member_is_preferred_over_better_member_needing_grace() { + let env = SelectionEnv::new(&[("korath", 1), ("solis", 1)]); + let _blocker = env.hold("korath", 0); + let acquired = acquire_group_member(&env.app_state, env.request(&[], true)).await.expect("acquired"); + assert_eq!((acquired.index, acquired.tier), (1, CapacityTier::Free)); +} + +#[tokio::test] +async fn lost_race_on_free_reprobes_free_before_grace() { + let env = SelectionEnv::new(&[("solis", 1), ("korath", 1), ("veyra", 1)]); + let _veyra_full = env.hold("veyra", 0); + let acquired = acquire_group_member_with_race(&env.app_state, env.request(&[], true), |state| { + let handle = state.active_provider.acquire_connection(&"solis".into(), &addr(201), 0, ConnectionKind::Normal); + std::mem::forget(handle); + }) + .await + .expect("acquired"); + assert_eq!((acquired.index, acquired.tier), (1, CapacityTier::Free)); +} + +#[tokio::test] +async fn own_release_keeps_best_member_when_zapping() { + let env = SelectionEnv::new(&[("korath", 1), ("solis", 1)]); + let own = env.hold("korath", 0); + env.app_state.active_users.set_pending_release_for_test("viewer", &[own.allocation_id]).await; + let acquired = acquire_group_member(&env.app_state, env.request(&[], true)).await.expect("acquired"); + assert_eq!((acquired.index, acquired.tier), (0, CapacityTier::OwnRelease)); +} + +#[tokio::test] +async fn preemption_only_on_best_ranked_when_nothing_free() { + let env = SelectionEnv::new(&[("korath", 1), ("solis", 1)]); + let _a = env.hold("korath", 10); + let _b = env.hold("solis", 10); + let mut req = env.request(&[], false); + req.priority = -5; + let acquired = acquire_group_member(&env.app_state, req).await.expect("acquired"); + assert_eq!((acquired.index, acquired.tier), (0, CapacityTier::Preempt)); +} + +#[tokio::test] +async fn demoted_input_members_are_tried_last() { + let env = SelectionEnv::new(&[("korath", 1), ("solis", 1)]); + let demoted: [Arc; 1] = ["korath".into()]; + let mut req = env.request(&[], true); + req.demoted_inputs = &demoted; + let acquired = acquire_group_member(&env.app_state, req).await.expect("acquired"); + assert_eq!((acquired.index, acquired.tier), (1, CapacityTier::Free)); +} + +#[tokio::test] +async fn grace_tier_skipped_when_disabled() { + let env = SelectionEnv::new(&[("korath", 1)]); + let _full = env.hold("korath", 0); + let result = acquire_group_member(&env.app_state, env.request(&[], false)).await; + assert!(matches!(result, Err(GroupSelectionFailure::Exhausted))); +} + +#[tokio::test] +async fn all_members_exhausted_returns_exhausted_within_budget() { + let mut env = SelectionEnv::new(&[("korath", 1), ("solis", 1)]); + env.config.selection.startup_budget_secs = 1; + let _a = env.hold("korath", 10); + let _a_grace = env + .app_state + .active_provider + .try_acquire_connection_no_preempt(&"korath".into(), true, &addr(201), 10, ConnectionKind::Normal, None) + .expect("grace"); + let _b = env.hold("solis", 10); + let _b_grace = env + .app_state + .active_provider + .try_acquire_connection_no_preempt(&"solis".into(), true, &addr(202), 10, ConnectionKind::Normal, None) + .expect("grace"); + let before = env.slots_in_use(); + let started = tokio::time::Instant::now(); + let mut req = env.request(&[], true); + req.priority = 10; + let result = acquire_group_member(&env.app_state, req).await; + assert!(matches!(result, Err(GroupSelectionFailure::Exhausted))); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(env.slots_in_use(), before); +} diff --git a/backend/app/src/api/api_utils/tests/channel_group_sticky.rs b/backend/app/src/api/api_utils/tests/channel_group_sticky.rs new file mode 100644 index 000000000..290e0a1e3 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/channel_group_sticky.rs @@ -0,0 +1,70 @@ +use super::group_playback_support::GroupPlaybackEnv; +use axum::http::StatusCode; +use futures::StreamExt; + +#[tokio::test] +async fn range_request_uses_bound_member_input_not_representative() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).with_qos("korath", 2, 95).build().await; + let initial = env.get_xtream_live(env.group_vid).await; + assert_eq!(initial.status(), StatusCode::OK); + assert_eq!(env.bound_member_input("viewer").await, "korath"); + assert_eq!(env.origin_hits("korath"), 1); + assert_eq!(env.origin_hits("solis"), 0); + + let seek = env.get_xtream_live_with_range(env.group_vid, "bytes=1000-").await; + assert!(seek.status().is_success()); + assert_eq!(env.bound_member_input("viewer").await, "korath"); + assert_eq!(env.origin_hits("korath"), 2); + assert_eq!(env.origin_hits("solis"), 0); +} + +#[tokio::test] +async fn ts_reconnect_stays_on_member_unless_circuit_open() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).with_qos("korath", 2, 95).build().await; + let first = env.get_xtream_live(env.group_vid).await; + assert_eq!(first.status(), StatusCode::OK); + drop(first); + + // reconnect within session TTL stays on korath + let second = env.get_xtream_live(env.group_vid).await; + assert_eq!(second.status(), StatusCode::OK); + drop(second); + assert_eq!((env.origin_hits("korath"), env.origin_hits("solis")), (2, 0)); + assert_eq!(env.bound_member_input("viewer").await, "korath"); + + // Open circuit on korath -> sticky binding is broken by circuit breaker, fails over to solis + env.open_circuit("korath"); + let third = env.get_xtream_live(env.group_vid).await; + assert_eq!(third.status(), StatusCode::OK); + drop(third); + assert_eq!(env.origin_hits("korath"), 2); + assert_eq!(env.origin_hits("solis"), 1); + assert_eq!(env.bound_member_input("viewer").await, "solis"); +} + +#[tokio::test] +async fn binding_survives_group_removal_until_stream_end() { + let env = GroupPlaybackEnv::builder(&["solis", "korath"]).build().await; + let response = env.get_xtream_live(env.group_vid).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(env.bound_member_input("viewer").await, "solis"); + assert!(env.active_session_vids("viewer").await.contains(&env.group_vid)); + + let mut body = response.into_body().into_data_stream(); + + // Republish target storage with groups removed while stream is actively running + env.republish_without_groups().await; + + // Running stream body continues delivering bytes despite group removal from storage + let chunk = body.next().await.expect("stream continues").expect("bytes"); + assert!(!chunk.is_empty()); + assert_eq!(env.bound_member_input("viewer").await, "solis"); + + // New request for the removed group VID sees new state and does not panic (returns 502 Bad Gateway / channel unavailable) + let after = env.get_xtream_live(env.group_vid).await; + assert_ne!(after.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(after.status(), StatusCode::BAD_GATEWAY); + + // After body finishes/is dropped, active session lease cleans up + drop(body); +} diff --git a/backend/app/src/api/api_utils/tests/cleanup.rs b/backend/app/src/api/api_utils/tests/cleanup.rs index f2aacb0a7..0244f856a 100644 --- a/backend/app/src/api/api_utils/tests/cleanup.rs +++ b/backend/app/src/api/api_utils/tests/cleanup.rs @@ -5,7 +5,7 @@ use super::{ create_test_provider_app_state, create_test_shared_target, force_provider_stream_response, get_stream_options, load_test_channel, load_test_session, load_test_user, session_reacquire_cleanup_addrs, spawn_controlled_fake_origin, spawn_range_aware_test_origin, stream_response, FakeOriginMode, - ForceStreamRequestContext, PlaybackRequestClass, SessionActivationRequest, StreamResponseMode, + ForceStreamRequestContext, PlaybackRequestClass, PlaybackSource, SessionActivationRequest, StreamResponseMode, }; use crate::{ api::model::{create_channel_unavailable_stream, UserSession}, @@ -1025,10 +1025,9 @@ async fn shared_proxy_clients_keep_independent_bodies_and_cleanup() { "shared-session-1", None, channel1, - &stream_url, + PlaybackSource::Single { input: &input, stream_url: &stream_url }, None, &HeaderMap::default(), - &input, &target, &user1, UserConnectionPermission::Allowed, @@ -1046,10 +1045,9 @@ async fn shared_proxy_clients_keep_independent_bodies_and_cleanup() { "shared-session-2", None, channel2, - &stream_url, + PlaybackSource::Single { input: &input, stream_url: &stream_url }, None, &HeaderMap::default(), - &input, &target, &user2, UserConnectionPermission::Allowed, @@ -1136,10 +1134,9 @@ async fn shared_start_abort_does_not_leak_meter_registration() { "shared-session-abort", None, channel, - &stream_url, + PlaybackSource::Single { input: &input, stream_url: &stream_url }, None, &HeaderMap::default(), - &input, &target, &user, UserConnectionPermission::Allowed, diff --git a/backend/app/src/api/api_utils/tests/group_playback_support.rs b/backend/app/src/api/api_utils/tests/group_playback_support.rs new file mode 100644 index 000000000..961e5b676 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/group_playback_support.rs @@ -0,0 +1,933 @@ +use super::support::FakeOriginMode; +use crate::{ + api::{ + api_utils::channel_group::{member_key, CircuitView}, + endpoints::{ + hdhomerun_api::hdhr_api_register, m3u_api::m3u_api_register, xmltv_api::xmltv_api_register, + xtream_api::xtream_api_register, + }, + model::{AppState, HdHomerunAppState}, + }, + model::{ + Config, ConfigInput, ConfigProvider, ConfigTarget, Epg, HdHomeRunDeviceConfig, HdHomeRunTargetOutput, + M3uTargetOutput, ProxyUserCredentials, SourcesConfig, StreamConfig, TargetOutput, XtreamTargetOutput, + }, + repository::{ + get_target_storage_path, persist_playlist_views_with_groups, ChannelGroupOrigin, ChannelGroupRecord, + ChannelGroupStore, PendingChannelGroup, PlaylistStorageState, TargetPlaylistPersistOptions, + }, +}; +use arc_swap::ArcSwap; +use axum::{ + body::Body, + http::{HeaderMap, Request}, + response::Response, +}; +use shared::{ + foundation::Filter, + model::{ + AdmissionStrategy, ChannelGroupPolicy, ChannelGroupSelectionConfig, ChannelGroupsConfig, ClusterFlags, + ConfigPaths, ConfigTargetOptions, HdHomeRunTargetOutputDto, InputType, LiveStreamProperties, + M3uTargetOutputDto, PlaylistItem, PlaylistItemHeader, PlaylistItemType, ProcessingOrder, ProxyUserStatus, + StreamProperties, TargetType, UUIDType, VirtualId, XtreamCluster, XtreamTargetOutputDto, + }, + utils::Internable, +}; +use std::{ + collections::HashMap, + net::SocketAddr, + str::FromStr, + sync::{atomic::AtomicI8, Arc}, +}; +use tempfile::TempDir; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; +use tower::ServiceExt; + +pub(super) fn ts_fixture() -> Vec { + let mut data = vec![0u8; 188 * 4]; + for i in 0..4 { + data[i * 188] = 0x47; + data[i * 188 + 1] = 0x1f; + data[i * 188 + 2] = 0xff; + data[i * 188 + 3] = 0x10; + } + data +} + +pub(crate) struct TrackedOrigin { + pub addr: SocketAddr, + pub requests: Arc>>, + pub _task: tokio::task::JoinHandle<()>, +} + +#[allow(clippy::too_many_lines)] +async fn spawn_tracked_origin(mode: FakeOriginMode) -> TrackedOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("test origin address"); + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let requests_clone = Arc::clone(&requests); + + let task = tokio::spawn(async move { + for _ in 0..20 { + let Ok((mut socket, _)) = listener.accept().await else { break }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let Ok(read) = socket.read(&mut chunk).await else { break }; + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + if request.is_empty() { + continue; + } + let request_str = String::from_utf8_lossy(&request).to_string(); + requests_clone.lock().expect("mutex").push(request_str.clone()); + + match &mode { + FakeOriginMode::EmptyBody => { + let head = + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::BlockFirstChunk(notify) => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let mut probe = [0_u8; 1]; + tokio::select! { + () = notify.notified() => {}, + _ = socket.read(&mut probe) => {}, + } + let _ = socket.shutdown().await; + } + FakeOriginMode::ErrorBeforeFirstChunk => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::ErrorAfterFirstChunk(first_chunk) => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let chunk_header = format!("{:x}\r\n", first_chunk.len()); + let _ = socket.write_all(chunk_header.as_bytes()).await; + let _ = socket.write_all(first_chunk).await; + let _ = socket.write_all(b"\r\n").await; + let _ = socket.shutdown().await; + } + FakeOriginMode::FixedBytes(bytes) => { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(bytes).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::Status(status) => { + let head = format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::Body { content_type, bytes } => { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(bytes).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::Hls => { + if request_str.contains(".ts") { + let bytes = ts_fixture(); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(&bytes).await; + let _ = socket.shutdown().await; + } else { + let body = "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:10.0,\nsegment1.ts\n#EXTINF:10.0,\nsegment2.ts\n"; + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.apple.mpegurl\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(body.as_bytes()).await; + let _ = socket.shutdown().await; + } + } + } + } + }); + + TrackedOrigin { addr, requests, _task: task } +} + +#[allow(dead_code)] +pub(crate) fn first_segment_url(manifest: &str) -> String { + manifest + .lines() + .map(str::trim) + .find(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_string) + .unwrap_or_default() +} + +#[allow(dead_code)] +pub(crate) struct GroupPlaybackEnv { + pub app_state: Arc, + pub target: Arc, + pub group_vid: u32, + pub plain_vid: u32, + pub favourite_vid: u32, + pub group_record: Arc, + pub origins: HashMap, + pub router: axum::Router, + pub hdhr_router: Option, + pub _dir: TempDir, +} + +#[allow(dead_code)] +impl GroupPlaybackEnv { + #[allow(clippy::new_ret_no_self)] + pub fn new(inputs: &[&str]) -> GroupPlaybackEnvBuilder { Self::builder(inputs) } + + pub fn builder(inputs: &[&str]) -> GroupPlaybackEnvBuilder { + GroupPlaybackEnvBuilder { + inputs: inputs.iter().map(|s| (*s).to_string()).collect(), + origin_modes: HashMap::new(), + qos: Vec::new(), + hls: false, + stalker: None, + user_limit: None, + admission: None, + favourite: false, + plain_channel: true, + member_archives: Vec::new(), + epg: None, + epg_channel_id: None, + hdhomerun: false, + } + } + + pub async fn get_xtream_live_streams(&self) -> Response { + let mut req = Request::builder() + .uri("/player_api.php?username=viewer&password=secret&action=get_live_streams") + .body(Body::empty()) + .expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn get_m3u_playlist(&self) -> Response { + let mut req = Request::builder() + .uri("/get.php?username=viewer&password=secret&type=m3u_plus&output=ts") + .body(Body::empty()) + .expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn get_hdhomerun_lineup(&self) -> Response { + let Some(ref hdhr_router) = self.hdhr_router else { + panic!("HDHomeRun router not initialized; did you call .with_hdhomerun()?"); + }; + let mut req = Request::builder().uri("/lineup.json").body(Body::empty()).expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + hdhr_router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn get_xmltv_epg(&self) -> Response { + let mut req = Request::builder() + .uri("/xmltv.php?username=viewer&password=secret") + .body(Body::empty()) + .expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn response_text(response: Response) -> String { + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + String::from_utf8(bytes.to_vec()).expect("utf-8 body") + } + + pub async fn get_xtream_live(&self, vid: u32) -> Response { + let mut req = Request::builder() + .uri(format!("/live/viewer/secret/{vid}.ts")) + .body(Body::empty()) + .expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn get_xtream_live_with_range(&self, vid: u32, range: &str) -> Response { + let mut req = Request::builder() + .uri(format!("/live/viewer/secret/{vid}.ts")) + .header("Range", range) + .body(Body::empty()) + .expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn get_m3u_live(&self, vid: u32) -> Response { + let mut req = Request::builder() + .uri(format!("/m3u-stream/live/viewer/secret/{vid}.ts")) + .body(Body::empty()) + .expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub fn origin_hits(&self, input: &str) -> usize { + self.origins.get(input).map_or(0, |o| o.requests.lock().expect("mutex").len()) + } + + pub fn last_request_headers(&self, input: &str) -> HeaderMap { + let mut map = HeaderMap::new(); + let Some(origin) = self.origins.get(input) else { return map }; + let guard = origin.requests.lock().expect("mutex"); + let Some(last) = guard.last() else { return map }; + for line in last.lines().skip(1) { + if line.is_empty() { + break; + } + if let Some((name, val)) = line.split_once(':') { + if let (Ok(n), Ok(v)) = + (axum::http::HeaderName::from_str(name.trim()), axum::http::HeaderValue::from_str(val.trim())) + { + map.insert(n, v); + } + } + } + map + } + + pub fn circuit(&self, input: &str) -> CircuitView { + let member = + self.group_record.members.iter().find(|m| m.input_name.as_ref() == input).expect("member for input"); + let key = member_key(member); + self.app_state.member_health.circuit_view(&key, std::time::Instant::now()) + } + + pub fn input_circuit(&self, input: &str) -> CircuitView { + self.app_state.member_health.input_circuit_view(input, std::time::Instant::now()) + } + + pub fn open_circuit(&self, input: &str) { + let member = + self.group_record.members.iter().find(|m| m.input_name.as_ref() == input).expect("member for input"); + let key = member_key(member); + for _ in 0..10 { + if matches!(self.circuit(input), CircuitView::Open { .. }) { + break; + } + self.app_state.member_health.record_classified_failure( + &key, + tuliprox_session::channel_group::FailureClass::Stream, + std::time::Instant::now(), + ); + } + } + + pub fn member_vid(&self, input: &str) -> u32 { + let member = + self.group_record.members.iter().find(|m| m.input_name.as_ref() == input).expect("member for input"); + member.member_vid.get() + } + + pub fn favourite_vid(&self) -> u32 { self.favourite_vid } + + pub fn hold(&self, input: &str, priority: i8) -> Vec { + let mut handles = Vec::new(); + while let Some(h) = self.app_state.active_provider.acquire_connection( + &input.into(), + &SocketAddr::from(([127, 0, 0, 1], 200)), + priority, + tuliprox_session::active_provider_manager::ConnectionKind::Normal, + ) { + handles.push(h); + } + handles + } + + pub async fn get_hls_manifest(&self, vid: u32) -> Response { + let mut req = Request::builder() + .uri(format!("/live/viewer/secret/{vid}.m3u8")) + .body(Body::empty()) + .expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn get_hls_manifest_text(&self, vid: u32) -> String { + let response = self.get_hls_manifest(vid).await; + assert_eq!(response.status(), 200); + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024).await.expect("bytes"); + String::from_utf8(bytes.to_vec()).expect("utf-8 manifest") + } + + pub async fn refresh_hls_manifest(&self, master_text: &str) -> Response { + let uri = first_segment_url(master_text); + self.get(&uri).await + } + + pub async fn get(&self, uri: &str) -> Response { + let path = if let Ok(parsed) = url::Url::parse(uri) { + let mut p = parsed.path().to_string(); + if let Some(stripped) = p.strip_prefix("/iptv") { + p = stripped.to_string(); + } + if let Some(q) = parsed.query() { + p.push('?'); + p.push_str(q); + } + p + } else if let Some(stripped) = uri.strip_prefix("/iptv") { + stripped.to_string() + } else { + uri.to_string() + }; + let mut req = Request::builder().uri(path).body(Body::empty()).expect("request builder"); + req.extensions_mut().insert(axum::extract::ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 12345)))); + self.router.clone().oneshot(req).await.expect("endpoint response") + } + + pub async fn bound_member_input(&self, username: &str) -> String { + self.app_state.active_users.bound_member_input(username).await.map(|s| s.to_string()).unwrap_or_default() + } + + pub async fn active_session_vids(&self, username: &str) -> Vec { + self.app_state.active_users.active_session_vids(username).await + } + + pub fn slots_in_use(&self) -> usize { self.app_state.active_provider.get_provider_connections_count() } + + pub async fn republish_without_groups(&self) { + let mut playlist = vec![shared::model::PlaylistGroup { + id: 1, + title: "News".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000002"), + id: "plain".intern(), + name: "Plain TV".intern(), + title: "Plain TV".intern(), + group: "News".intern(), + url: "http://127.0.0.1/plain".intern(), + input_name: self.inputs()[0].as_str().intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }], + xtream_cluster: XtreamCluster::Live, + }]; + persist_playlist_views_with_groups( + &self.app_state.app_config, + &mut playlist, + None, + None, + &self.target, + Some(&self.app_state.playlists), + TargetPlaylistPersistOptions::default(), + &mut [], + ) + .await + .expect("republished without groups"); + } + + fn inputs(&self) -> Vec { self.origins.keys().cloned().collect() } +} + +#[allow(clippy::struct_excessive_bools)] +pub(crate) struct GroupPlaybackEnvBuilder { + inputs: Vec, + origin_modes: HashMap, + qos: Vec<(String, u32, u8)>, + hls: bool, + stalker: Option, + user_limit: Option, + admission: Option>, + favourite: bool, + plain_channel: bool, + member_archives: Vec>, + epg: Option, + epg_channel_id: Option, + hdhomerun: bool, +} + +#[allow(dead_code)] +impl GroupPlaybackEnvBuilder { + pub fn origin_status(mut self, input: &str, status: u16) -> Self { + self.origin_modes.insert(input.to_string(), FakeOriginMode::Status(status)); + self + } + + pub fn origin_body(mut self, input: &str, content_type: &str, body: Vec) -> Self { + self.origin_modes + .insert(input.to_string(), FakeOriginMode::Body { content_type: content_type.to_string(), bytes: body }); + self + } + + pub fn with_qos(mut self, input: &str, provider_id: u32, score: u8) -> Self { + self.qos.push((input.to_string(), provider_id, score)); + self + } + + pub fn with_favourite(mut self) -> Self { + self.favourite = true; + self + } + + pub fn hls(mut self) -> Self { + self.hls = true; + self + } + + pub fn stalker(mut self, input: &str) -> Self { + self.stalker = Some(input.to_string()); + self + } + + pub fn user_limit(mut self, limit: u32) -> Self { + self.user_limit = Some(limit); + self + } + + pub fn admission(mut self, strategies: Vec) -> Self { + self.admission = Some(strategies); + self + } + + pub fn without_plain_channel(mut self) -> Self { + self.plain_channel = false; + self + } + + pub fn with_member_archives(mut self, archives: &[Option]) -> Self { + self.member_archives = archives.to_vec(); + self + } + + pub fn with_epg(mut self, epg: Epg, channel_id: &str) -> Self { + self.epg = Some(epg); + self.epg_channel_id = Some(channel_id.to_string()); + self + } + + pub fn with_hdhomerun(mut self) -> Self { + self.hdhomerun = true; + self + } + + #[allow(clippy::too_many_lines)] + pub async fn build(self) -> GroupPlaybackEnv { + let dir = TempDir::new().expect("temp dir"); + + let mut origins = HashMap::new(); + for input in &self.inputs { + let mode = self.origin_modes.get(input).cloned().unwrap_or_else(|| { + if self.hls { + FakeOriginMode::Hls + } else { + FakeOriginMode::FixedBytes(ts_fixture()) + } + }); + let origin = spawn_tracked_origin(mode).await; + origins.insert(input.clone(), origin); + } + + let mut config_inputs = Vec::new(); + for input_name in &self.inputs { + let origin = &origins[input_name]; + let is_stalker = self.stalker.as_deref() == Some(input_name.as_str()); + let provider = ConfigProvider { + name: Arc::from(input_name.as_str()), + urls: vec![Arc::from(format!("http://{}", origin.addr).as_str())], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + current_url_index: std::sync::atomic::AtomicUsize::new(0), + dns_cache: Arc::default(), + }; + let input = Arc::new(ConfigInput { + id: u16::try_from(config_inputs.len()).unwrap_or(0) + 1, + name: Arc::from(input_name.as_str()), + input_type: if is_stalker { InputType::Stalker } else { InputType::Xtream }, + url: format!("http://{}", origin.addr), + username: Some("viewer".to_string()), + password: Some("secret".to_string()), + priority: 1, + max_connections: 10, + enabled: true, + provider_configs: Some(vec![Arc::new(provider)]), + stalker: if is_stalker { + Some(crate::model::StalkerInputConfig { + device: None, + auth_mode: shared::model::StalkerAuthMode::Auto, + mag_preset: shared::model::StalkerMagPreset::GenericSafe, + endpoint_preference: shared::model::StalkerEndpointPreference::ServerLoad, + size_caps: None, + catalog_max_pages: None, + username: None, + password: None, + }) + } else { + None + }, + ..ConfigInput::default() + }); + config_inputs.push(input); + } + + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + user.password = "secret".to_string(); + user.status = Some(ProxyUserStatus::Active); + user.output_clusters = ClusterFlags::all(); + user.max_connections = self.user_limit.unwrap_or(2); + user.proxy = shared::model::ProxyType::Reverse(None); + + let storage_dir = dir.path().to_string_lossy().into_owned(); + let config = Config { + storage_dir: storage_dir.clone(), + user_access_control: true, + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(StreamConfig { admission_strategies: self.admission, ..StreamConfig::default() }), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + }; + + let target_name = "test_target".to_string(); + let api_proxy = crate::model::ApiProxyConfig { + server: vec![crate::model::ApiProxyServerInfo { + name: "default".to_string(), + protocol: "https".to_string(), + host: "example.test".to_string(), + port: None, + timezone: "UTC".to_string(), + message: String::new(), + path: Some("iptv".to_string()), + }], + user: vec![crate::model::TargetUser { target: target_name.clone(), credentials: vec![Arc::new(user)] }], + ..Default::default() + }; + + let mut target_outputs = vec![ + TargetOutput::Xtream(XtreamTargetOutput::from(&XtreamTargetOutputDto::default())), + TargetOutput::M3u(M3uTargetOutput::from(&M3uTargetOutputDto { + filename: Some("playlist.m3u".to_string()), + ..M3uTargetOutputDto::default() + })), + ]; + if self.hdhomerun { + target_outputs.push(TargetOutput::HdHomeRun(HdHomeRunTargetOutput::from(&HdHomeRunTargetOutputDto { + device: "test_hdhr".to_string(), + username: "viewer".to_string(), + use_output: Some(TargetType::M3u), + }))); + } + + let target = Arc::new(ConfigTarget { + id: 1, + enabled: true, + name: "test_target".to_string(), + options: Some(ConfigTargetOptions { + channel_groups: Some(ChannelGroupsConfig { + enabled: true, + selection: ChannelGroupSelectionConfig::default(), + ..ChannelGroupsConfig::default() + }), + ..ConfigTargetOptions::default() + }), + output: target_outputs, + filter: Filter::default().into(), + sort: None, + curation: None, + rename: None, + mapping_ids: None, + mapping: Arc::new(arc_swap::ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }); + + let mut app_cfg = crate::api::api_utils::tests::support::create_test_app_config(); + app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); + app_cfg.api_proxy = Arc::new(arc_swap::ArcSwapOption::from(Some(Arc::new(api_proxy)))); + app_cfg.sources = Arc::new(ArcSwap::from_pointee(SourcesConfig { + inputs: config_inputs.clone(), + sources: vec![crate::model::ConfigSource { + inputs: config_inputs.iter().map(|i| Arc::clone(&i.name)).collect(), + targets: vec![Arc::clone(&target)], + }], + ..SourcesConfig::default() + })); + app_cfg.paths = Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: storage_dir.clone(), + config_path: storage_dir.clone(), + storage_path: storage_dir, + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })); + + let group_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000001"); + let representative_input = &self.inputs[0]; + let representative_origin = &origins[representative_input]; + let rep_url = format!("http://{}/live/101", representative_origin.addr); + + let group_epg = self.epg_channel_id.as_deref().map(Internable::intern); + let max_archive = self.member_archives.iter().flatten().copied().max(); + let group_additional_props = max_archive.map(|dur| { + StreamProperties::Live(Box::new(LiveStreamProperties { + name: "ZDF HD".intern(), + category_id: 1, + stream_id: 0, + stream_icon: "".intern(), + direct_source: "".intern(), + custom_sid: None, + added: None, + stream_type: None, + epg_channel_id: group_epg.clone(), + tv_archive: Some(1), + tv_archive_duration: Some(dur), + is_adult: 0, + ..Default::default() + })) + }); + + let group_channel = PlaylistItem { + header: PlaylistItemHeader { + uuid: group_uuid, + id: "zdf-hd".intern(), + input_stream_id: "101".intern(), + name: "ZDF HD".intern(), + title: "ZDF HD".intern(), + group: "General".intern(), + url: rep_url.as_str().intern(), + input_name: Arc::from(representative_input.as_str()), + item_type: if self.hls { PlaylistItemType::LiveHls } else { PlaylistItemType::Live }, + xtream_cluster: XtreamCluster::Live, + channel_group_uuid: Some(group_uuid), + epg_channel_id: group_epg.clone(), + additional_properties: group_additional_props, + ..PlaylistItemHeader::default() + }, + }; + + let plain_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000002"); + let plain_channel = PlaylistItem { + header: PlaylistItemHeader { + uuid: plain_uuid, + id: "plain-tv".intern(), + input_stream_id: "plain".intern(), + name: "Plain TV".intern(), + title: "Plain TV".intern(), + group: "General".intern(), + url: format!("http://{}/live/plain", representative_origin.addr).as_str().intern(), + input_name: Arc::from(representative_input.as_str()), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + + let mut members = Vec::new(); + for (idx, input_name) in self.inputs.iter().enumerate() { + let origin = &origins[input_name]; + let member_url = format!("http://{}/live/101", origin.addr); + let member_archive = self.member_archives.get(idx).copied().flatten(); + let member_additional_props = member_archive.map(|dur| { + StreamProperties::Live(Box::new(LiveStreamProperties { + name: format!("ZDF HD ({input_name})").intern(), + category_id: 1, + stream_id: 0, + stream_icon: "".intern(), + direct_source: "".intern(), + custom_sid: None, + added: None, + stream_type: None, + epg_channel_id: group_epg.clone(), + tv_archive: Some(1), + tv_archive_duration: Some(dur), + is_adult: 0, + ..Default::default() + })) + }); + let member = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid(&format!("00000000-0000-4000-8000-0000000000{:02x}", idx + 10)), + id: (idx + 1).to_string().intern(), + input_stream_id: (idx + 1).to_string().intern(), + name: format!("ZDF HD ({input_name})").intern(), + title: "ZDF HD".intern(), + group: "General".intern(), + url: member_url.as_str().intern(), + input_name: Arc::from(input_name.as_str()), + item_type: if self.hls { PlaylistItemType::LiveHls } else { PlaylistItemType::Live }, + xtream_cluster: XtreamCluster::Live, + epg_channel_id: group_epg.clone(), + additional_properties: member_additional_props, + ..PlaylistItemHeader::default() + }, + }; + members.push(member); + } + + let channels = + if self.plain_channel { vec![group_channel.clone(), plain_channel] } else { vec![group_channel.clone()] }; + + let mut playlist = vec![shared::model::PlaylistGroup { + id: 1, + title: "General".intern(), + channels, + xtream_cluster: XtreamCluster::Live, + }]; + + let mut favourite_vid = 0; + if self.favourite { + let fav_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000099"); + let fav_channel = PlaylistItem { + header: PlaylistItemHeader { + uuid: fav_uuid, + channel_group_uuid: Some(group_uuid), + group: "Favourites".intern(), + ..group_channel.header.clone() + }, + }; + playlist.push(shared::model::PlaylistGroup { + id: 99, + title: "Favourites".intern(), + channels: vec![fav_channel], + xtream_cluster: XtreamCluster::Live, + }); + } + + let mut pending = vec![PendingChannelGroup { + group_uuid, + key: Arc::from("zdf hd"), + origin: ChannelGroupOrigin::Auto, + policy: ChannelGroupPolicy::Auto, + members, + orphaned: Vec::new(), + epg_split: false, + }]; + + let app_cfg = Arc::new(app_cfg); + let playlists = Arc::new(PlaylistStorageState::new()); + persist_playlist_views_with_groups( + &app_cfg, + &mut playlist, + None, + self.epg.as_ref(), + &target, + Some(&playlists), + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .expect("persists playlist views"); + + let group_vid = playlist[0].channels[0].header.virtual_id.get(); + let plain_vid = if self.plain_channel { playlist[0].channels[1].header.virtual_id.get() } else { 0 }; + if self.favourite { + favourite_vid = playlist[1].channels[0].header.virtual_id.get(); + } + + let config_snapshot = app_cfg.config.load(); + let path = get_target_storage_path(&config_snapshot, &target.name).expect("path"); + let record = + ChannelGroupStore::read_group(&path, VirtualId::new(group_vid)).expect("read").expect("record exists"); + let group_record = Arc::new(record); + + let app_state = crate::api::api_utils::tests::support::create_test_app_state_for_config(Arc::clone(&app_cfg)); + if let Some(cfg) = target.options.as_ref().and_then(|o| o.channel_groups.as_ref()).map(|cg| &cg.selection) { + app_state.member_health.set_policy(tuliprox_session::channel_group::CircuitPolicy::from(cfg)); + } + app_state + .member_health + .set_event_sink(Arc::clone(&app_state.event_manager) as Arc); + + for (input, provider_id, score) in self.qos { + app_state.member_health.replace_qos([( + tuliprox_session::channel_group::QosKey::Provider { input: Arc::from(input.as_str()), provider_id }, + tuliprox_session::channel_group::QosPart { + score_24h: score, + score_7d: score, + confidence: 100, + failure_streak: 0, + }, + )]); + } + + let router = xtream_api_register() + .merge(m3u_api_register()) + .merge(crate::api::endpoints::hls_api::hls_api_register()) + .merge(xmltv_api_register()) + .with_state(Arc::clone(&app_state)); + + let hdhr_router = if self.hdhomerun { + let hdhr_state = Arc::new(HdHomerunAppState { + app_state: Arc::clone(&app_state), + device: Arc::new(HdHomeRunDeviceConfig { + friendly_name: "Tuliprox HDHR".to_string(), + manufacturer: "Tuliprox".to_string(), + model_name: "HDHR-TEST".to_string(), + model_number: "HDHR4-2US".to_string(), + firmware_name: "hdhomerun4_atsc".to_string(), + firmware_version: "20260101".to_string(), + device_id: "12345678".to_string(), + device_type: "tuner".to_string(), + device_udn: "uuid:12345678-0000-0000-0000-000000000000".to_string(), + name: "test_hdhr".to_string(), + port: 5004, + tuner_count: 2, + t_username: "viewer".to_string(), + t_enabled: true, + }), + hd_scan_state: Arc::new(AtomicI8::new(-1)), + }); + Some(hdhr_api_register(false).with_state(hdhr_state)) + } else { + None + }; + + GroupPlaybackEnv { + app_state, + target, + group_vid, + plain_vid, + favourite_vid, + group_record, + origins, + router, + hdhr_router, + _dir: dir, + } + } +} diff --git a/backend/app/src/api/api_utils/tests/maintenance.rs b/backend/app/src/api/api_utils/tests/maintenance.rs index 81612f2c6..ebcb45dfb 100644 --- a/backend/app/src/api/api_utils/tests/maintenance.rs +++ b/backend/app/src/api/api_utils/tests/maintenance.rs @@ -1,6 +1,6 @@ use super::{ create_test_app_state_for_config, create_test_fingerprint, create_test_live_channel, - create_test_provider_app_config, create_test_provider_app_state, stream_response, + create_test_provider_app_config, create_test_provider_app_state, stream_response, PlaybackSource, }; use crate::{ api::{api_utils::blocks_new_playback, model::AppState}, @@ -172,10 +172,9 @@ async fn new_ts_stream_for_input_in_maintenance_gets_the_clip_without_provider_s "maintenance-session", None, create_test_live_channel(stream_url), - stream_url, + PlaybackSource::Single { input: &input, stream_url }, None, &HeaderMap::default(), - &input, &maintenance_clip_target(), &viewer(), shared::model::UserConnectionPermission::Allowed, diff --git a/backend/app/src/api/api_utils/tests/provider_urls.rs b/backend/app/src/api/api_utils/tests/provider_urls.rs index 4cb92f4e3..f3908033f 100644 --- a/backend/app/src/api/api_utils/tests/provider_urls.rs +++ b/backend/app/src/api/api_utils/tests/provider_urls.rs @@ -872,6 +872,7 @@ async fn xtream_vod_m3u_reverse_proxies_external_sources_and_alias_redirects_wit Some("mp4"), None, None, + None, ) .await .into_response(); diff --git a/backend/app/src/api/api_utils/tests/stream_responses_response_headers.rs b/backend/app/src/api/api_utils/tests/stream_responses_response_headers.rs index 26ed21ec4..91bf3c8ec 100644 --- a/backend/app/src/api/api_utils/tests/stream_responses_response_headers.rs +++ b/backend/app/src/api/api_utils/tests/stream_responses_response_headers.rs @@ -4,7 +4,7 @@ use super::{ is_media_server_playback_url, is_media_server_stream_ref_url, is_seek_request, is_seekable_media_request, mark_response_as_uncompressed, resolve_request_url_for_logging, resolve_stream_config_u64, resource_proxy_response, spawn_legacy_hls_test_origin, stream_response, stream_responses::forced_legacy_hls_test_response, - user_with_network_access, NetworkAccessDecision, + user_with_network_access, NetworkAccessDecision, PlaybackSource, }; use crate::{ api::model::{SharedStreamCtx, SharedStreamManager}, @@ -365,10 +365,9 @@ async fn stream_response_preserves_soft_kind_for_shared_reuse() { "soft-session", None, create_test_live_channel(stream_url), - stream_url, + PlaybackSource::Single { input: &input, stream_url }, None, &HeaderMap::default(), - &input, &target, &user, admission.permission(), diff --git a/backend/app/src/api/api_utils/tests/stream_responses_session_activation.rs b/backend/app/src/api/api_utils/tests/stream_responses_session_activation.rs index a191a6223..c106b7db6 100644 --- a/backend/app/src/api/api_utils/tests/stream_responses_session_activation.rs +++ b/backend/app/src/api/api_utils/tests/stream_responses_session_activation.rs @@ -2,7 +2,7 @@ use super::{ activate_session_before_stream_open, create_test_app_config, create_test_app_state, create_test_app_state_for_config, create_test_app_state_with_stream_config, create_test_fingerprint, create_test_live_channel, create_test_provider_app_config, resolve_admission_with_strategies, stream_response, - AdmissionRequest, EvictionReentryGuard, PlaybackRequestClass, SessionActivationRequest, + AdmissionRequest, EvictionReentryGuard, PlaybackRequestClass, PlaybackSource, SessionActivationRequest, }; use crate::model::{Config, ConfigTarget, ProxyUserCredentials}; use arc_swap::{ArcSwap, ArcSwapOption}; @@ -573,10 +573,9 @@ async fn stream_response_rolls_back_provisional_user_activation_when_provider_op "rollback-session", None, channel, - stream_url, + PlaybackSource::Single { input: &input, stream_url }, None, &HeaderMap::default(), - &input, &target, &user, UserConnectionPermission::Allowed, diff --git a/backend/app/src/api/api_utils/tests/support.rs b/backend/app/src/api/api_utils/tests/support.rs index 387f4f5a3..0447bf48a 100644 --- a/backend/app/src/api/api_utils/tests/support.rs +++ b/backend/app/src/api/api_utils/tests/support.rs @@ -110,6 +110,9 @@ pub(in crate::api::api_utils::tests) enum FakeOriginMode { ErrorBeforeFirstChunk, ErrorAfterFirstChunk(Vec), FixedBytes(Vec), + Status(u16), + Body { content_type: String, bytes: Vec }, + Hls, } pub(in crate::api::api_utils::tests) async fn spawn_controlled_fake_origin( @@ -178,6 +181,25 @@ pub(in crate::api::api_utils::tests) async fn spawn_controlled_fake_origin( let _ = socket.write_all(bytes).await; let _ = socket.shutdown().await; } + FakeOriginMode::Status(status) => { + let head = format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::Body { content_type, bytes } => { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(bytes).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::Hls => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.apple.mpegurl\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } } } requests @@ -470,6 +492,7 @@ pub(in crate::api::api_utils::tests) fn create_test_app_state_for_config(app_cfg auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), maintenance: crate::api::model::create_test_maintenance(""), + member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()), }) } @@ -507,6 +530,7 @@ pub(in crate::api::api_utils::tests) fn create_test_local_channel(url: &str) -> StreamChannel { target_id: 1, virtual_id: 41, + group_virtual_id: None, provider_id: 0, input_name: "library".intern(), item_type: PlaylistItemType::LocalVideo, @@ -528,6 +552,7 @@ pub(in crate::api::api_utils::tests) fn create_test_live_channel(url: &str) -> S StreamChannel { target_id: 1, virtual_id: 42, + group_virtual_id: None, provider_id: 1, input_name: "provider_1".intern(), item_type: PlaylistItemType::Live, diff --git a/backend/app/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs index 068f0721b..720d1b482 100644 --- a/backend/app/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -738,6 +738,7 @@ mod tests { auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), maintenance: crate::api::model::create_test_maintenance(""), + member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()), }) } diff --git a/backend/app/src/api/endpoints/hls_api/manifest/cached.rs b/backend/app/src/api/endpoints/hls_api/manifest/cached.rs index 0b7e2a3dc..87d7c77de 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/cached.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/cached.rs @@ -155,10 +155,16 @@ pub(in crate::api::endpoints::hls_api) fn spawn_hls_runtime_bandwidth_persistenc let app_config = Arc::clone(&app_state.app_config); let hls_proxy = Arc::clone(&app_state.hls.proxy); let session = Arc::clone(session); + let member_health = Arc::clone(&app_state.member_health); Some(tokio::spawn(async move { let outcome = match persist_input_live_bitrate_bps(&app_config, &input, &stream_ref, bitrate_bps).await { - Ok(repository_outcome) => hls_bandwidth_persistence_outcome(repository_outcome, &proxy_session_id), + Ok(repository_outcome) => { + let kbps = bitrate_bps / 1000; + let key = (Arc::clone(&input.name), Arc::from(stream_ref.as_ref())); + member_health.patch_probe(&key, None, kbps); + hls_bandwidth_persistence_outcome(repository_outcome, &proxy_session_id) + } Err(err) => { error!( "HLS runtime bandwidth persistence failed: proxy_session={} error={}", diff --git a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs index 3820bc4d7..9cbf2d0f1 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs @@ -113,7 +113,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_reserve_hls_entry_origin_acc }) } -#[allow(clippy::too_many_arguments)] +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(in crate::api::endpoints::hls_api) async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( app_state: &Arc, fingerprint: &Fingerprint, @@ -168,6 +168,49 @@ pub(in crate::api::endpoints::hls_api) async fn try_reserve_hls_virtual_entry_or }; let session_owner = shared_hls_session_owner.as_deref().unwrap_or(session_token.as_str()); + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + VirtualId::new(virtual_id), + ) + .await; + if let tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) = classified { + let (selection_cfg, allow_provider_grace) = + target.options.as_ref().and_then(|o| o.channel_groups.as_ref()).map_or_else( + || (shared::model::ChannelGroupSelectionConfig::default(), true), + |cg| (cg.selection, cg.allow_provider_grace), + ); + let policy = group.policy; + let target_name: Arc = Arc::from(target.name.as_str()); + + let req = crate::api::api_utils::GroupSelectionRequest { + target_name: &target_name, + group: &group, + policy, + mode: tuliprox_session::channel_group::SelectionMode::Live, + selection: selection_cfg, + addr: &fingerprint.addr, + username: &user.username, + priority: connection_priority_for_kind(user, connection_kind), + kind: connection_kind, + lease: Some(PlaybackLeaseRef::new(session_owner, PlaybackKind::LiveHls)), + allow_provider_grace, + exclude: &[], + demoted_inputs: &[], + prefer: None, + }; + let Ok(resolved) = + crate::api::api_utils::resolve_hls_group_entry(app_state, target, Arc::clone(&group), req).await + else { + return false; + }; + let mut acquired_handle = resolved.acquired.handle; + let provider_handle = acquired_handle.take(); + app_state.connection_manager.release_provider_handle(provider_handle); + return true; + } + let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( app_state, fingerprint, diff --git a/backend/app/src/api/endpoints/hls_api/owner_token.rs b/backend/app/src/api/endpoints/hls_api/owner_token.rs index ff2a825ec..7b414c4e5 100644 --- a/backend/app/src/api/endpoints/hls_api/owner_token.rs +++ b/backend/app/src/api/endpoints/hls_api/owner_token.rs @@ -185,9 +185,33 @@ async fn hls_recreate_content_allowed( return Some(user_allows_entry_content(user, scope, &item)); } } - let item = - m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target).await.ok()?; - Some(user_allows_entry_content(user, HlsEntryOutputScope::ItemType(item.item_type), &item)) + if let Ok(item) = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target).await + { + return Some(user_allows_entry_content(user, HlsEntryOutputScope::ItemType(item.item_type), &item)); + } + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + shared::model::VirtualId::new(virtual_id), + ) + .await; + if let tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } = classified { + if target.has_output(TargetType::Xtream) { + let item = tuliprox_repository::target_stream_item::member_to_xtream_item(&group, index, target); + let scope = if is_catchup { + HlsEntryOutputScope::LiveCluster + } else { + HlsEntryOutputScope::ItemType(item.item_type) + }; + return Some(user_allows_entry_content(user, scope, &item)); + } + let item = tuliprox_repository::target_stream_item::member_to_m3u_item(&group, index, target); + let scope = + if is_catchup { HlsEntryOutputScope::LiveCluster } else { HlsEntryOutputScope::ItemType(item.item_type) }; + return Some(user_allows_entry_content(user, scope, &item)); + } + None } /// Recreates a missing user session from a sealed `Manifest(Entry)` token. @@ -195,7 +219,7 @@ async fn hls_recreate_content_allowed( /// With the wrapper the player refreshes only the variant URI, so after a session expiry the /// token route has to do what an entry refresh did before: identity, access, admission, source, /// session, fetch. The session keeps the sealed session token and therefore its playback owner. -#[allow(clippy::too_many_arguments)] +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(super) async fn recreate_hls_session_from_token( fingerprint: &Fingerprint, req_headers: &HeaderMap, @@ -211,9 +235,21 @@ pub(super) async fn recreate_hls_session_from_token( else { return StatusCode::BAD_REQUEST.into_response(); }; - if hls_cache_enabled_for_user(app_state, target, user) - || !hls_session_hint_matches_requester(fingerprint, &user.username, virtual_id, hint) - { + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + shared::model::VirtualId::new(virtual_id), + ) + .await; + let hint_matches = hls_session_hint_matches_requester(fingerprint, &user.username, virtual_id, hint) + || match &classified { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, .. } => { + hls_session_hint_matches_requester(fingerprint, &user.username, group.group_vid.get(), hint) + } + _ => false, + }; + if hls_cache_enabled_for_user(app_state, target, user) || !hint_matches { return StatusCode::BAD_REQUEST.into_response(); } // Only an expired session comes back; an evicted, kicked or terminated one stays ended, diff --git a/backend/app/src/api/endpoints/hls_api/segment/entry.rs b/backend/app/src/api/endpoints/hls_api/segment/entry.rs index 073eda608..7b0e01ca5 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/entry.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/entry.rs @@ -5,15 +5,16 @@ use super::{ hls_custom_video_manifest_response, hls_entry_user_session_token, hls_media_playlist_wrap_enabled, hls_panel_provisioning_or_status_response, hls_playback_kind, hls_response, m3u_archive_epg_reference_ts, m3u_catchup_epg_reference_from_session_token, normalize_xtream_live_hls_url, release_prepared_hls_manifest_session, - terminate_failed_hls_manifest_session, try_reserve_hls_entry_origin_account_for_redirect, wrap_media_playlist, - HlsEntryStreamContext, HlsMediaPlaylistWrap, HlsOriginEntryUrl, HlsRequestStage, + resolve_hls_origin_playlist_url, terminate_failed_hls_manifest_session, + try_reserve_hls_entry_origin_account_for_redirect, wrap_media_playlist, HlsEntryStreamContext, + HlsMediaPlaylistWrap, HlsOriginEntryUrl, HlsRequestStage, }; use crate::{ api::{ api_utils::{ acquire_exact_provider_handle, blocks_new_playback, connection_priority_for_kind, - get_hls_playback_ttl_secs, is_running_playback, select_provider_stream_url, ExactProviderAcquire, - HLS_MANIFEST_CAPACITY_WAIT, + get_hls_playback_ttl_secs, is_running_playback, resolve_hls_group_entry, select_provider_stream_url, + ExactProviderAcquire, GroupSelectionRequest, HLS_MANIFEST_CAPACITY_WAIT, }, model::{AppState, CustomVideoStreamType, PlaybackLeaseRef, ProviderAllocation, UserSession}, }, @@ -40,6 +41,10 @@ use tuliprox_hls::api::{ build_hls_origin_session_owner, build_proxy_session_id, extract_hls_provider_session_headers, HlsOriginSource, HlsSessionKey, }; +use tuliprox_session::{ + channel_group::{FailureClass, HalfOpenGuard, SelectionMode}, + MemberBinding, +}; pub(in crate::api::endpoints::hls_api) struct HlsCacheManifestOrigin<'a> { pub(in crate::api::endpoints::hls_api) raw_request_url: &'a str, @@ -120,7 +125,7 @@ pub(in crate::api) async fn handle_hls_stream_request( }; // The wrapper seals the normalized entry URL, not the provider-resolved one, so every // refresh can resolve it to whichever account is selected then. - let entry_url = url.clone(); + let mut entry_url = url.clone(); // Recover archive context when callers (esp. Xtream timeshift) pass None but the // resolved provider URL / catchup session still carries Flussonic archive markers. let archive_reference = archive_reference.or_else(|| m3u_archive_epg_reference_ts(&url)).or_else(|| { @@ -175,8 +180,21 @@ pub(in crate::api) async fn handle_hls_stream_request( } let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); + let mut bound_member_info: Option<(u32, Arc, String, Option)> = None; + let (request_url, session_token, provider_handle, selected_provider_config) = if let Some(session) = user_session { - let pinned_provider = if session.provider.is_empty() { &input.name } else { &session.provider }; + let bound_inp_opt = session.member_binding.as_ref().and_then(|binding| { + app_state.app_config.get_input_by_name(&binding.input_name).map(|inp| (binding.member_vid, inp)) + }); + if let Some((member_vid, ref bound_inp)) = bound_inp_opt { + bound_member_info = Some((member_vid, Arc::clone(bound_inp), String::new(), None)); + } + let effective_lookup_input = bound_inp_opt.as_ref().map_or(input, |(_, inp)| inp.as_ref()); + let pinned_provider = if session.provider.is_empty() { + effective_lookup_input.name.clone() + } else { + Arc::clone(&session.provider) + }; let pinned_kind = hls_playback_kind(archive_reference, &url, Some(session.token.as_str())); let session_kind = session.connection_kind.or(connection_kind).unwrap_or(crate::api::model::ConnectionKind::Normal); @@ -185,7 +203,7 @@ pub(in crate::api) async fn handle_hls_stream_request( let acquired = acquire_exact_provider_handle( app_state, &ExactProviderAcquire { - provider: pinned_provider, + provider: &pinned_provider, addr: &fingerprint.addr, allow_grace: false, priority: connection_priority_for_kind(user, session_kind), @@ -200,7 +218,7 @@ pub(in crate::api) async fn handle_hls_stream_request( } else { debug_if_enabled!( "HLS pinned provider {} unavailable for {}; aborting allocation to prevent mid-session migration", - sanitize_sensitive_info(pinned_provider), + sanitize_sensitive_info(&pinned_provider), sanitize_sensitive_info(&fingerprint.addr.to_string()) ); None @@ -216,7 +234,7 @@ pub(in crate::api) async fn handle_hls_stream_request( return hls_panel_provisioning_or_status_response( app_state, user, - input, + effective_lookup_input, virtual_id, original_hls_entry_path, server_info.path.as_deref(), @@ -250,7 +268,7 @@ pub(in crate::api) async fn handle_hls_stream_request( }; let Some((_provider_name, stream_url)) = select_provider_stream_url( &url, - input, + effective_lookup_input, &selected_provider_config, accept_requested_stream_url, &app_state.app_config, @@ -277,7 +295,7 @@ pub(in crate::api) async fn handle_hls_stream_request( debug_if_enabled!( "HLS session {} changed while waiting for pinned provider {}", sanitize_sensitive_info(&session.token), - sanitize_sensitive_info(pinned_provider) + sanitize_sensitive_info(&pinned_provider) ); return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); } @@ -291,7 +309,7 @@ pub(in crate::api) async fn handle_hls_stream_request( .create_user_session(crate::api::model::CreateUserSessionParams { user, session_token: &session.token, - virtual_id, + virtual_id: session.virtual_id, provider: &selected_provider_config.name, stream_url: &stream_url, addr: &fingerprint.addr, @@ -312,67 +330,207 @@ pub(in crate::api) async fn handle_hls_stream_request( None => (url, None, None, None), } } else { - // Append/shift catchup must keep an m3u-catchup session token even when shared HLS - // cache is off; otherwise rewritten segments register as LiveHls in the panel. - let user_session_token = hls_entry_user_session_token( - fingerprint, - &user.username, - virtual_id, - session_token_hint, - archive_reference, - ); - let hls_session_owner = if hls_cache_enabled_for_user(app_state, target, user) { - let session_key = HlsSessionKey::new(input.id, stream_context.stream_ref()); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - Some(build_hls_origin_session_owner(&proxy_session_id)) - } else { - None - }; - let session_owner = hls_session_owner.as_deref().unwrap_or(user_session_token.as_str()); - // Archive playback keeps its own reconnect window semantics, so the lease must - // be classified as catchup rather than plain live HLS. - let playback_kind = hls_playback_kind(archive_reference, &url, Some(user_session_token.as_str())); - let hls_session_ttl_secs = get_hls_playback_ttl_secs(app_state, playback_kind); - let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( - app_state, - fingerprint, - user, - input, - virtual_id, - &url, - &user_session_token, - session_owner, - playback_kind, - hls_session_ttl_secs, - connection_permission, - fallback_connection_kind, - true, + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + VirtualId::new(virtual_id), ) - .await - else { - return hls_panel_provisioning_or_status_response( + .await; + + if let tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) = classified { + let user_session_token = hls_entry_user_session_token( + fingerprint, + &user.username, + virtual_id, + session_token_hint, + archive_reference, + ); + let playback_kind = hls_playback_kind(archive_reference, &url, Some(user_session_token.as_str())); + let hls_session_ttl_secs = get_hls_playback_ttl_secs(app_state, playback_kind); + + let (selection_cfg, allow_provider_grace) = + target.options.as_ref().and_then(|o| o.channel_groups.as_ref()).map_or_else( + || (shared::model::ChannelGroupSelectionConfig::default(), true), + |cg| (cg.selection, cg.allow_provider_grace), + ); + let policy = group.policy; + let target_name: Arc = Arc::from(target.name.as_str()); + + let req = GroupSelectionRequest { + target_name: &target_name, + group: &group, + policy, + mode: SelectionMode::Live, + selection: selection_cfg, + addr: &fingerprint.addr, + username: &user.username, + priority: connection_priority_for_kind(user, fallback_connection_kind), + kind: fallback_connection_kind, + lease: Some(PlaybackLeaseRef::new(user_session_token.as_str(), playback_kind)), + allow_provider_grace, + exclude: &[], + demoted_inputs: &[], + prefer: None, + }; + + let Ok(resolved) = resolve_hls_group_entry(app_state, target, Arc::clone(&group), req).await else { + return hls_panel_provisioning_or_status_response( + app_state, + user, + input, + virtual_id, + original_hls_entry_path, + server_info.path.as_deref(), + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + }; + + let mut acquired_handle = resolved.acquired.handle; + let provider_handle = acquired_handle.take(); + let selected_provider_config = + provider_handle.as_ref().and_then(|handle| handle.allocation.get_provider_config()); + + let origin_member_url = resolve_hls_origin_playlist_url( app_state, + target, + &resolved.input, + resolved.member.member_vid.get(), + resolved.stream_url.as_ref(), + ) + .await + .unwrap_or_else(|_| resolved.stream_url.to_string()); + + let member_url = if let Some(cfg) = &selected_provider_config { + select_provider_stream_url(&origin_member_url, &resolved.input, cfg, false, &app_state.app_config) + .await + .map_or_else(|| origin_member_url.clone(), |(_, u)| u) + } else { + origin_member_url.clone() + }; + + let provider_name = selected_provider_config + .as_ref() + .map_or_else(|| Arc::clone(&resolved.input.name), |cfg| Arc::clone(&cfg.name)); + + let session_token = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token: &user_session_token, + virtual_id, + provider: &provider_name, + stream_url: &member_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(fallback_connection_kind), + socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), + }) + .await; + + app_state + .active_users + .bind_member( + &user.username, + &session_token, + MemberBinding { + group_vid: group.group_vid.get(), + member_vid: resolved.member.member_vid.get(), + member_index: resolved.acquired.index, + input_name: Arc::clone(&resolved.member.input_name), + }, + ) + .await; + + app_state.active_provider.refresh_adaptive_playback_lease( + &provider_name, + &user_session_token, + playback_kind, + hls_session_ttl_secs, + ); + + bound_member_info = Some(( + resolved.member.member_vid.get(), + resolved.input, + resolved.member.input_stream_id.to_string(), + resolved.acquired.half_open, + )); + + debug_if_enabled!( + "API endpoint [HLS] create_session_fingerprint user={} virtual_id={virtual_id} (group) member_vid={} provider={} stream_url={}", + sanitize_sensitive_info(&user.username), + resolved.member.member_vid.get(), + selected_provider_config.as_ref().map_or("", |provider| provider.name.as_ref()), + sanitize_sensitive_info(&member_url) + ); + + entry_url = member_url.clone(); + + (member_url, Some(session_token), provider_handle, selected_provider_config) + } else { + // Append/shift catchup must keep an m3u-catchup session token even when shared HLS + // cache is off; otherwise rewritten segments register as LiveHls in the panel. + let user_session_token = hls_entry_user_session_token( + fingerprint, + &user.username, + virtual_id, + session_token_hint, + archive_reference, + ); + let hls_session_owner = if hls_cache_enabled_for_user(app_state, target, user) { + let session_key = HlsSessionKey::new(input.id, stream_context.stream_ref()); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + Some(build_hls_origin_session_owner(&proxy_session_id)) + } else { + None + }; + let session_owner = hls_session_owner.as_deref().unwrap_or(user_session_token.as_str()); + // Archive playback keeps its own reconnect window semantics, so the lease must + // be classified as catchup rather than plain live HLS. + let playback_kind = hls_playback_kind(archive_reference, &url, Some(user_session_token.as_str())); + let hls_session_ttl_secs = get_hls_playback_ttl_secs(app_state, playback_kind); + let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( + app_state, + fingerprint, user, input, virtual_id, - original_hls_entry_path, - server_info.path.as_deref(), - StatusCode::SERVICE_UNAVAILABLE, + &url, + &user_session_token, + session_owner, + playback_kind, + hls_session_ttl_secs, + connection_permission, + fallback_connection_kind, + true, ) - .await; - }; - debug_if_enabled!( - "API endpoint [HLS] create_session_fingerprint user={} virtual_id={virtual_id} provider={} stream_url={}", - sanitize_sensitive_info(&user.username), - reservation.selected_provider_config.as_ref().map_or("", |provider| provider.name.as_ref()), - sanitize_sensitive_info(&reservation.request_url) - ); - ( - reservation.request_url, - Some(reservation.session_token), - reservation.provider_handle, - reservation.selected_provider_config, - ) + .await + else { + return hls_panel_provisioning_or_status_response( + app_state, + user, + input, + virtual_id, + original_hls_entry_path, + server_info.path.as_deref(), + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + }; + debug_if_enabled!( + "API endpoint [HLS] create_session_fingerprint user={} virtual_id={virtual_id} provider={} stream_url={}", + sanitize_sensitive_info(&user.username), + reservation.selected_provider_config.as_ref().map_or("", |provider| provider.name.as_ref()), + sanitize_sensitive_info(&reservation.request_url) + ); + ( + reservation.request_url, + Some(reservation.session_token), + reservation.provider_handle, + reservation.selected_provider_config, + ) + } }; // The session as created or reserved above; manifest response cookies may only enter this @@ -403,9 +561,16 @@ pub(in crate::api) async fn handle_hls_stream_request( } } + let (effective_virtual_id, effective_input, effective_stream_ref, half_open_guard) = + if let Some((m_vid, ref m_input, ref m_sref, guard)) = bound_member_info { + (m_vid, m_input.as_ref(), if m_sref.is_empty() { stream_ref.as_str() } else { m_sref.as_str() }, guard) + } else { + (virtual_id, input, stream_ref.as_str(), None) + }; + let user_agent_stream_index = crate::api::api_utils::resolve_stream_user_agent_index( app_state, - input, + effective_input, provider_handle.is_some(), &user.username, session_token.as_deref(), @@ -426,7 +591,7 @@ pub(in crate::api) async fn handle_hls_stream_request( // breaks forced same-account reuse on the next HLS/Catchup stream request. app_state.connection_manager.release_provider_handle(provider_handle); - let input_source = InputSource::from(input).with_url(request_url); + let input_source = InputSource::from(effective_input).with_url(request_url); let download_result = match download_legacy_hls_manifest(app_state, &input_source, &headers).await { Ok((content, response_url, response_headers)) => match classify_hls_playlist(&content) { // A refresh keeps its session on a malformed body; the player retries the playlist. @@ -447,6 +612,9 @@ pub(in crate::api) async fn handle_hls_stream_request( }; match download_result { Ok((content, response_url, response_headers, playlist_kind)) => { + if let Some(guard) = half_open_guard { + app_state.member_health.record_success_with(guard); + } let encrypt_secret = app_state.get_encrypt_secret(); let base_url = server_info.get_base_url(); let rewrite_hls_props = RewriteHlsProps { @@ -455,8 +623,8 @@ pub(in crate::api) async fn handle_hls_stream_request( content: &content, hls_url: response_url, target_id: target.id, - virtual_id, - input_id: input.id, + virtual_id: effective_virtual_id, + input_id: effective_input.id, user_token: session_token.as_deref(), origin_provider: selected_provider_name.as_deref(), playlist_kind: Some(playlist_kind), @@ -489,14 +657,14 @@ pub(in crate::api) async fn handle_hls_stream_request( user, base_url: &base_url, target_id: target.id, - input, - virtual_id, + input: effective_input, + virtual_id: effective_virtual_id, session_token, sealed_url: &entry_url, provider, binding_tag: provider_binding_tag, known_bitrate_bps: stream_context.known_bitrate_bps(), - stream_ref: Some(&stream_ref), + stream_ref: Some(effective_stream_ref), }, hls_content, ) @@ -507,6 +675,13 @@ pub(in crate::api) async fn handle_hls_stream_request( hls_response(hls_content).into_response() } Err(err) => { + if let Some(guard) = half_open_guard { + app_state.member_health.record_classified_failure_with( + guard, + FailureClass::Stream, + std::time::Instant::now(), + ); + } error!("Failed to download m3u8: {}", request::text_response_error_log_label(&err)); if let Some(session_token) = session_token.as_deref() { terminate_failed_hls_manifest_session( @@ -544,8 +719,28 @@ pub(in crate::api::endpoints::hls_api) async fn get_stream_channel( } } let target_id = target.id; - m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) - .await - .ok() - .map(|pli| pli.to_stream_channel(target_id)) + if let Ok(pli) = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target).await { + return Some(pli.to_stream_channel(target_id)); + } + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + VirtualId::new(virtual_id), + ) + .await; + match classified { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } => { + let mut channel = if target.has_output(TargetType::Xtream) { + tuliprox_repository::target_stream_item::member_to_xtream_item(&group, index, target) + .to_stream_channel(target_id) + } else { + tuliprox_repository::target_stream_item::member_to_m3u_item(&group, index, target) + .to_stream_channel(target_id) + }; + channel.group_virtual_id = Some(group.group_vid.get()); + Some(channel) + } + _ => None, + } } diff --git a/backend/app/src/api/endpoints/hls_api/segment/registration.rs b/backend/app/src/api/endpoints/hls_api/segment/registration.rs index 935159283..505a04d2e 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/registration.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/registration.rs @@ -181,6 +181,7 @@ pub(in crate::api::endpoints::hls_api) fn fallback_hls_cache_stream_channel( StreamChannel { target_id, virtual_id, + group_virtual_id: None, provider_id: 0, input_name: Arc::clone(&origin_source.input_name), item_type: PlaylistItemType::LiveHls, diff --git a/backend/app/src/api/endpoints/hls_api/segment/routing.rs b/backend/app/src/api/endpoints/hls_api/segment/routing.rs index 4ccf9b7b0..8c40ddbf0 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/routing.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/routing.rs @@ -31,7 +31,7 @@ use shared::{ defaults::HLS_EXT, model::{ ConnectFailureReason, PlaylistEntry, PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, - XtreamCluster, + VirtualId, XtreamCluster, }, utils::{generate_random_string, is_m3u_catchup_session_token, sanitize_sensitive_info, Internable}, }; @@ -58,19 +58,47 @@ pub(in crate::api) async fn resolve_hls_virtual_source_for_target( { let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; (Arc::clone(&item.input_name), stream_context) - } else { - let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) - .await - .map_err(|_| StatusCode::NOT_FOUND)?; + } else if let Ok(item) = + m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target).await + { let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; (Arc::clone(&item.input_name), stream_context) + } else { + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + VirtualId::new(virtual_id), + ) + .await; + if let tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } = classified { + let item = tuliprox_repository::target_stream_item::member_to_xtream_item(&group, index, target); + let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_context) + } else { + return Err(StatusCode::NOT_FOUND); + } } - } else { - let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) - .await - .map_err(|_| StatusCode::NOT_FOUND)?; + } else if let Ok(item) = + m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target).await + { let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; (Arc::clone(&item.input_name), stream_context) + } else { + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + VirtualId::new(virtual_id), + ) + .await; + if let tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } = classified { + let item = tuliprox_repository::target_stream_item::member_to_m3u_item(&group, index, target); + let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_context) + } else { + return Err(StatusCode::NOT_FOUND); + } }; let input = app_state.app_config.get_input_by_name(&input_name).ok_or(StatusCode::NOT_FOUND)?; Ok(HlsResolvedVirtualSource { input, stream_context }) @@ -78,15 +106,30 @@ pub(in crate::api) async fn resolve_hls_virtual_source_for_target( pub(in crate::api::endpoints::hls_api) async fn resolve_hls_origin_playlist_url( app_state: &Arc, - target: &Arc, + target: &ConfigTarget, input: &ConfigInput, virtual_id: u32, fallback_url: &str, ) -> Result { if input.input_type.is_xtream() && target.has_output(TargetType::Xtream) { - let pli = xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None) - .await - .map_err(|_| StatusCode::NOT_FOUND)?; + let pli = if let Ok(item) = + xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await + { + item + } else { + let classified = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + target, + VirtualId::new(virtual_id), + ) + .await; + if let tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } = classified { + tuliprox_repository::target_stream_item::member_to_xtream_item(&group, index, target) + } else { + return Err(StatusCode::NOT_FOUND); + } + }; let hls_extension = format!(".{HLS_EXT}"); let (query_path, _) = get_query_path("", Some(&hls_extension), &pli, app_state); return get_xtream_player_api_stream_url(input, ApiStreamContext::Live, &query_path, &pli.url) @@ -133,6 +176,7 @@ pub(in crate::api::endpoints::hls_api) async fn resolve_stream_channel( None => StreamChannel { target_id: target.id, virtual_id, + group_virtual_id: None, provider_id: 0, input_name: Arc::clone(&input.name), item_type: PlaylistItemType::LiveHls, @@ -417,7 +461,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_api_stream_resolved( let archive_reference = resolve_m3u_archive_reference(&hls_url, Some(session.token.as_str())) .or_else(|| m3u_archive_epg_reference_ts(session.stream_url.as_ref())); session.stream_url = hls_url.clone(); - if session.virtual_id == virtual_id { + if session.virtual_id == virtual_id + || session.member_binding.as_ref().is_some_and(|b| b.member_vid == virtual_id) + { app_state.connection_manager.touch_http_activity(&user.username, &session.token, &fingerprint.addr).await; } else { return axum::http::StatusCode::BAD_REQUEST.into_response(); diff --git a/backend/app/src/api/endpoints/hls_api/tests.rs b/backend/app/src/api/endpoints/hls_api/tests.rs index 7d0a36fec..0a005f5f6 100644 --- a/backend/app/src/api/endpoints/hls_api/tests.rs +++ b/backend/app/src/api/endpoints/hls_api/tests.rs @@ -138,3 +138,5 @@ mod terminal_tail_recovery_before_cutover; mod terminal_tail_terminal_cutover; mod terminal_tail_terminal_fallback; mod terminal_tail_terminal_preflight; + +mod channel_group; diff --git a/backend/app/src/api/endpoints/hls_api/tests/channel_group.rs b/backend/app/src/api/endpoints/hls_api/tests/channel_group.rs new file mode 100644 index 000000000..ed8ffcd08 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/channel_group.rs @@ -0,0 +1,38 @@ +use crate::api::api_utils::tests::group_playback_support::{first_segment_url, GroupPlaybackEnv}; +use axum::http::StatusCode; + +#[tokio::test] +async fn hls_entry_selects_member_with_tiers() { + let env = GroupPlaybackEnv::new(&["solis", "korath"]).hls().build().await; + let _blocker = env.hold("solis", 0); + let manifest = env.get_hls_manifest(env.group_vid).await; + assert_eq!(manifest.status(), StatusCode::OK); + assert_eq!(env.origin_hits("solis"), 0); + assert_eq!(env.origin_hits("korath"), 1); + assert_eq!(env.bound_member_input("viewer").await, "korath"); +} + +#[tokio::test] +async fn segment_routing_resolves_member_vid_to_member_input() { + let env = GroupPlaybackEnv::new(&["solis", "korath"]).hls().with_qos("korath", 2, 95).build().await; + let manifest = env.get_hls_manifest_text(env.group_vid).await; + let segment_url = first_segment_url(&manifest); + assert!(segment_url.contains(&env.member_vid("korath").to_string())); + let segment = env.get(&segment_url).await; + assert_eq!(segment.status(), StatusCode::OK); + assert_eq!(env.origin_hits("solis"), 0); + assert_eq!(env.origin_hits("korath"), 1); + assert_eq!(env.bound_member_input("viewer").await, "korath"); +} + +#[tokio::test] +async fn manifest_refresh_never_reselects_even_with_open_circuit() { + let env = GroupPlaybackEnv::new(&["solis", "korath"]).hls().build().await; + let first = env.get_hls_manifest_text(env.group_vid).await; + let bound = env.bound_member_input("viewer").await; + assert!(!bound.is_empty()); + env.open_circuit(&bound); + let second = env.refresh_hls_manifest(&first).await; + assert_eq!(second.status(), StatusCode::OK); + assert_eq!(env.bound_member_input("viewer").await, bound); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs index b464e1b64..5a678d435 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs @@ -285,6 +285,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_app_state_with_hls_proxy_a auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), maintenance: crate::api::model::create_test_maintenance(""), + member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()), }) } diff --git a/backend/app/src/api/endpoints/m3u_api.rs b/backend/app/src/api/endpoints/m3u_api.rs index a6a7f2001..af71851a2 100644 --- a/backend/app/src/api/endpoints/m3u_api.rs +++ b/backend/app/src/api/endpoints/m3u_api.rs @@ -9,7 +9,7 @@ use crate::{ resolve_initial_stalker_playback_url, resource_proxy_response, resource_redirect_or_proxy, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response_with_provider_handle, try_option_bad_request, try_result_bad_request, try_result_not_found, - try_unwrap_body, RedirectParams, + try_unwrap_body, PlaybackSource, RedirectParams, }, endpoints::{ hls_api::{ @@ -155,6 +155,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( stream_ext: Option<&str>, archive_discriminator: Option<&str>, provider_allocation_id: Option, + source: Option>, ) -> impl IntoResponse + Send { let target_name = &target.name; if !target.has_output(TargetType::M3u) { @@ -183,7 +184,30 @@ pub(in crate::api) async fn m3u_api_stream_loaded( } let virtual_id = pli.virtual_id; - if app_state.active_users.is_user_blocked_for_stream(&user.username, virtual_id).await { + let resolved_target = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + &target, + virtual_id, + ) + .await; + let is_group = source.as_ref().map_or_else( + || { + matches!( + resolved_target, + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(_) + | tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { .. } + ) + }, + |s| matches!(s, PlaybackSource::Group { .. }), + ); + + let check_vid = match &resolved_target { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) + | tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, .. } => group.group_vid, + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Item => virtual_id, + }; + if app_state.active_users.is_user_blocked_for_stream(&user.username, check_vid).await { return axum::http::StatusCode::BAD_REQUEST.into_response(); } @@ -249,23 +273,25 @@ pub(in crate::api) async fn m3u_api_stream_loaded( } let cluster = pli.item_type.cluster(); - pli.url = match resolve_initial_stalker_playback_url( - app_state, - &input, - &user, - pli.get_provider_id().unwrap_or_default(), - cluster, - pli.item_type, - &pli.url, - ) - .await - { - Ok(url) => url, - Err(err) => { - error!("Failed to resolve initial Stalker playback URL: {}", sanitize_sensitive_info(&err.to_string())); - return axum::http::StatusCode::BAD_GATEWAY.into_response(); - } - }; + if !is_group { + pli.url = match resolve_initial_stalker_playback_url( + app_state, + &input, + &user, + pli.get_provider_id().unwrap_or_default(), + cluster, + pli.item_type, + &pli.url, + ) + .await + { + Ok(url) => url, + Err(err) => { + error!("Failed to resolve initial Stalker playback URL: {}", sanitize_sensitive_info(&err.to_string())); + return axum::http::StatusCode::BAD_GATEWAY.into_response(); + } + }; + } let stream_channel = || { let mut channel = pli.to_stream_channel(target.id); @@ -355,11 +381,13 @@ pub(in crate::api) async fn m3u_api_stream_loaded( } if session.virtual_id == virtual_id.get() && is_seekable_media_request(cluster, req_headers, Some(extension)) { // partial request means we are in reverse proxy mode, seek happened + let (input, stream_channel) = + crate::api::api_utils::sticky_playback_context(app_state, session, &input, stream_channel()); return force_provider_stream_response( fingerprint, app_state, session, - stream_channel(), + stream_channel, crate::api::api_utils::ForceStreamRequestContext { req_headers, input: &input, @@ -527,16 +555,47 @@ pub(in crate::api) async fn m3u_api_stream_loaded( None }; + let bound_prefer = user_session + .as_ref() + .and_then(|session| session.member_binding.as_ref()) + .map(|b| (b.group_vid, b.member_index)); + + let mut source = source.unwrap_or_else(|| match resolved_target { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) => { + let prefer = bound_prefer.filter(|(gvid, _)| *gvid == group.group_vid.get()).map(|(_, idx)| idx); + PlaybackSource::Group { group, mode: tuliprox_session::channel_group::SelectionMode::Live, prefer } + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } => { + PlaybackSource::Group { + group, + mode: tuliprox_session::channel_group::SelectionMode::Live, + prefer: Some(index), + } + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Item => { + PlaybackSource::Single { input: &input, stream_url: &session_url } + } + }); + + if let PlaybackSource::Group { group, prefer, .. } = &mut source { + if prefer.is_none() { + if let Some((gvid, idx)) = bound_prefer { + if gvid == group.group_vid.get() { + *prefer = Some(idx); + } + } + } + } + stream_response_with_provider_handle( fingerprint, app_state, &session_key, Some(request_class), stream_channel().with_epg_reference_ts(archive_reference), - &session_url, + source, pinned_provider, req_headers, - &input, &target, &user, connection_permission, @@ -716,6 +775,7 @@ async fn m3u_api_stream( Some(archive.extension()), Some(&discriminator), None, + None, ) .await .into_response(); @@ -723,10 +783,25 @@ async fn m3u_api_stream( let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id); let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); - let pli = - match m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target).await { - Ok(pli) => pli, - Err(err) => { + let resolved_target = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + &target, + shared::model::VirtualId::new(req_virtual_id), + ) + .await; + let pli_res = + m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target).await; + let pli = match pli_res { + Ok(pli) => pli, + Err(err) => match &resolved_target { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) => { + tuliprox_repository::target_stream_item::member_to_m3u_item(group, 0, &target) + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } => { + tuliprox_repository::target_stream_item::member_to_m3u_item(group, *index, &target) + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Item => { error!("Failed to read m3u item for stream id {req_virtual_id}: {err}"); if stream_ext == Some(HLS_EXT) { return axum::http::StatusCode::NOT_FOUND.into_response(); @@ -738,7 +813,8 @@ async fn m3u_api_stream( ) .into_response(); } - }; + }, + }; let input = try_option_bad_request!( app_state.app_config.get_input_by_name(&pli.input_name), @@ -756,6 +832,21 @@ async fn m3u_api_stream( } else { (pli, None) }; + let source = match resolved_target { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) => Some(PlaybackSource::Group { + group, + mode: tuliprox_session::channel_group::SelectionMode::Live, + prefer: None, + }), + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } => { + Some(PlaybackSource::Group { + group, + mode: tuliprox_session::channel_group::SelectionMode::Live, + prefer: Some(index), + }) + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Item => None, + }; m3u_api_stream_loaded( user, target, @@ -767,6 +858,7 @@ async fn m3u_api_stream( stream_ext, archive_discriminator.as_deref(), None, + source, ) .await .into_response() @@ -843,6 +935,7 @@ async fn m3u_api_stream_nested( stream_ext, archive_discriminator.as_deref(), None, + None, ) .await .into_response() @@ -911,6 +1004,7 @@ async fn m3u_api_catchup( None, Some(&archive_discriminator), None, + None, ) .await .into_response() diff --git a/backend/app/src/api/endpoints/v1_api.rs b/backend/app/src/api/endpoints/v1_api.rs index b19058625..08cef8ffc 100644 --- a/backend/app/src/api/endpoints/v1_api.rs +++ b/backend/app/src/api/endpoints/v1_api.rs @@ -430,6 +430,7 @@ mod tests { let channel = StreamChannel { target_id: 1, virtual_id: 70, + group_virtual_id: None, provider_id: 1, input_name: "input".intern(), item_type: PlaylistItemType::Series, diff --git a/backend/app/src/api/endpoints/v1_api_playlist/recording.rs b/backend/app/src/api/endpoints/v1_api_playlist/recording.rs index a5f8a6864..8ae54d9bc 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/recording.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/recording.rs @@ -313,6 +313,7 @@ pub(super) async fn playlist_recording_stream( None, None, query.provider_allocation_id, + None, ) .await .into_response() diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests.rs index 8897e75b6..17d6124a3 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests.rs @@ -17,3 +17,4 @@ mod behavior_epg_preview; mod behavior_manual_update; mod behavior_provider_resolution; mod behavior_recording_routes; +mod channel_groups; diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/channel_groups.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/channel_groups.rs new file mode 100644 index 000000000..950bddbbb --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/channel_groups.rs @@ -0,0 +1,101 @@ +use crate::{api::api_utils::tests::group_playback_support::GroupPlaybackEnv, model::Epg}; +use shared::{ + model::{EpgChannel, EpgProgramme}, + utils::Internable, +}; +use std::sync::Arc; + +fn test_epg(channel_id: &str, title: &str, programme_title: &str) -> Epg { + let mut programme = EpgProgramme::new(3600, 7200, channel_id.intern()); + programme.title = Some(programme_title.intern()); + Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![Arc::new(EpgChannel { + id: channel_id.intern(), + title: Some(title.intern()), + icon: None, + programmes: vec![programme], + })], + } +} + +#[tokio::test] +async fn channel_groups_outputs_and_epg_sanity() { + let epg = test_epg("zdf.de", "ZDF", "Heute Journal"); + let env = GroupPlaybackEnv::builder(&["solis", "korath"]) + .without_plain_channel() + .with_member_archives(&[Some(3), Some(7)]) + .with_epg(epg, "zdf.de") + .with_hdhomerun() + .build() + .await; + + let member_solis = env.member_vid("solis"); + let member_korath = env.member_vid("korath"); + let group_vid = env.group_vid; + + // 1. Xtream get_live_streams lists the group vid, no member vid, and tv_archive/tv_archive_duration equals member maximum + let res = env.get_xtream_live_streams().await; + assert_eq!(res.status(), 200); + let body_text = GroupPlaybackEnv::response_text(res).await; + let streams: serde_json::Value = serde_json::from_str(&body_text).expect("valid JSON array"); + let list = streams.as_array().expect("array of streams"); + assert_eq!(list.len(), 1, "Only group stream should be listed: {body_text}"); + assert_eq!(list[0]["stream_id"], group_vid); + assert_eq!(list[0]["name"], "ZDF HD"); + assert_eq!(list[0]["epg_channel_id"], "zdf.de"); + assert_eq!(list[0]["tv_archive"], 1); + assert_eq!(list[0]["tv_archive_duration"], 7); // maximum of 3 and 7 + assert!(!body_text.contains(&member_solis.to_string()), "must not contain member solis vid"); + assert!(!body_text.contains(&member_korath.to_string()), "must not contain member korath vid"); + + // 2. M3U output lists one entry with the group EPG id + let res = env.get_m3u_playlist().await; + assert_eq!(res.status(), 200); + let m3u_text = GroupPlaybackEnv::response_text(res).await; + assert_eq!(m3u_text.matches("#EXTINF:").count(), 1, "M3U must contain exactly one stream: {m3u_text}"); + assert!(m3u_text.contains(r#"tvg-id="zdf.de""#), "M3U must have group EPG id: {m3u_text}"); + assert!(m3u_text.contains(&format!("/{group_vid}")), "M3U stream URL must reference group vid: {m3u_text}"); + assert!(!m3u_text.contains(&format!("/{member_solis}")), "M3U must not contain member solis vid"); + assert!(!m3u_text.contains(&format!("/{member_korath}")), "M3U must not contain member korath vid"); + + // 3. HDHomeRun lineup lists one entry + let res = env.get_hdhomerun_lineup().await; + assert_eq!(res.status(), 200); + let lineup_text = GroupPlaybackEnv::response_text(res).await; + let lineup: serde_json::Value = serde_json::from_str(&lineup_text).expect("valid JSON array"); + let lineup_list = lineup.as_array().expect("array of lineup items"); + assert_eq!(lineup_list.len(), 1, "Lineup must contain exactly one item: {lineup_text}"); + assert_eq!(lineup_list[0]["GuideNumber"], "zdf.de"); + assert_eq!(lineup_list[0]["GuideName"], "ZDF HD"); + let lineup_url = lineup_list[0]["URL"].as_str().unwrap_or(""); + assert!( + lineup_url.contains(&group_vid.to_string()), + "Lineup URL must reference group vid: {lineup_url} vs group {group_vid}" + ); + assert!( + !lineup_url.ends_with(&format!("/{member_solis}")) && !lineup_url.ends_with(&format!("/{member_solis}.ts")), + "Lineup must not end with member solis vid: {lineup_url} vs solis {member_solis}" + ); + assert!( + !lineup_url.ends_with(&format!("/{member_korath}")) && !lineup_url.ends_with(&format!("/{member_korath}.ts")), + "Lineup must not end with member korath vid: {lineup_url} vs korath {member_korath}" + ); + + // 4. XMLTV contains the group EPG channel + let res = env.get_xmltv_epg().await; + assert_eq!(res.status(), 200); + let xmltv_text = GroupPlaybackEnv::response_text(res).await; + assert!(xmltv_text.contains(r#""#), "XMLTV must contain group EPG channel: {xmltv_text}"); + assert!( + xmltv_text.contains("ZDF HD"), + "XMLTV must contain channel display name: {xmltv_text}" + ); + assert!( + xmltv_text.contains(r#"channel="zdf.de""#), + "XMLTV must have programme referencing group EPG channel: {xmltv_text}" + ); + assert!(xmltv_text.contains("Heute Journal"), "XMLTV must contain programme title: {xmltv_text}"); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs index 09125f35a..629dc045f 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs @@ -172,5 +172,6 @@ pub(in crate::api::endpoints::v1_api_playlist::tests) fn test_app_state_with_man auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests_with_sender(manual_update_sender), maintenance: crate::api::model::create_test_maintenance(""), + member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()), }) } diff --git a/backend/app/src/api/endpoints/v1_api_playlist/webplayer.rs b/backend/app/src/api/endpoints/v1_api_playlist/webplayer.rs index ecc0045e5..a1f58d9e5 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/webplayer.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/webplayer.rs @@ -87,7 +87,7 @@ pub(super) async fn playlist_webplayer_stream( ); let user = Arc::new(create_api_proxy_user(&app_state)); - m3u_api_stream_loaded(user, target, &fingerprint, &req_headers, &app_state, pli, input, None, None, None) + m3u_api_stream_loaded(user, target, &fingerprint, &req_headers, &app_state, pli, input, None, None, None, None) .await .into_response() } diff --git a/backend/app/src/api/endpoints/websocket_api.rs b/backend/app/src/api/endpoints/websocket_api.rs index bd58467de..e1f178b00 100644 --- a/backend/app/src/api/endpoints/websocket_api.rs +++ b/backend/app/src/api/endpoints/websocket_api.rs @@ -540,7 +540,10 @@ fn to_protocol_message(event: EventMessage) -> Option<(ProtocolMessage, &'static // Auth decisions reach notification channels and plugins, not the // Web UI socket: there is no panel that renders them, and pushing // every sign-in to every connected admin is noise. - | EventMessage::AuthAudit(_) => return None, + | EventMessage::AuthAudit(_) + | EventMessage::ChannelGroupMemberSelected { .. } + | EventMessage::ChannelGroupExhausted { .. } + | EventMessage::ChannelGroupCircuitChanged { .. } => return None, }) } @@ -748,7 +751,7 @@ mod tests { // every notification-only kind against `expected = true` and the // test failed on `DiskAlert` - it has been red since the lifecycle // events joined the bus. - const NOT_ON_THE_WIRE: [EventKind; 28] = [ + const NOT_ON_THE_WIRE: [EventKind; 31] = [ EventKind::ConnectionDenied, EventKind::ScheduledTaskFailed, EventKind::ProviderFetchFailed, @@ -777,6 +780,9 @@ mod tests { EventKind::AuthSignInFailed, EventKind::AuthSignInThrottled, EventKind::AuthPermissionDenied, + EventKind::ChannelGroupMemberSelected, + EventKind::ChannelGroupExhausted, + EventKind::ChannelGroupCircuitChanged, ]; for (event, kind) in sample_event_of_every_kind() { @@ -941,6 +947,18 @@ mod tests { auth_audit(AuthAuditOutcome::SignInThrottled), auth_audit(AuthAuditOutcome::PermissionDenied), EventMessage::MaintenanceChanged(Arc::default()), + EventMessage::ChannelGroupMemberSelected { + target_id: 1, + group_vid: 10, + member_vid: 11, + tier: shared::model::channel_group::CapacityTier::Free, + }, + EventMessage::ChannelGroupExhausted { target_id: 1, group_vid: 10 }, + EventMessage::ChannelGroupCircuitChanged { + input_name: "test".into(), + input_stream_id: "1".into(), + state: shared::model::channel_group::CircuitState::Closed, + }, ]; assert_eq!(samples.len(), EventKind::ALL.len(), "add the new variant to this list"); samples diff --git a/backend/app/src/api/endpoints/xtream_api/mod.rs b/backend/app/src/api/endpoints/xtream_api/mod.rs index 962cd4d2b..0e6f18755 100644 --- a/backend/app/src/api/endpoints/xtream_api/mod.rs +++ b/backend/app/src/api/endpoints/xtream_api/mod.rs @@ -14,7 +14,7 @@ use crate::{ redirect, redirect_response, reentry_suppressed_response, resolve_initial_stalker_playback_url, resource_proxy_response, resource_redirect_or_proxy, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, stream_response_with_provider_handle, - try_option_bad_request, try_result_bad_request, try_unwrap_body, RedirectParams, + try_option_bad_request, try_result_bad_request, try_unwrap_body, PlaybackSource, RedirectParams, }, endpoints::{ hls_api::{ @@ -295,27 +295,53 @@ async fn xtream_player_api_stream( } let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); - let Ok(mut pli) = - xtream_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target, None).await - else { - error!("Failed to read xtream item for stream id {req_virtual_id}"); - if is_hls_manifest_request { - return hls_custom_video_manifest_response( - app_state, - &user, - CustomVideoStreamType::ChannelUnavailable, - axum::http::StatusCode::NOT_FOUND, - ) - .await; - } - return create_custom_video_stream_response( - &app_state.provider_stream_ctx(), - &fingerprint.addr, - CustomVideoStreamType::ChannelUnavailable, - ) - .into_response(); + let resolved_target = tuliprox_repository::target_stream_item::classify_virtual_id( + &app_state.app_config, + &app_state.playlists, + &target, + VirtualId::new(req_virtual_id), + ) + .await; + + let is_group = matches!( + resolved_target, + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(_) + | tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { .. } + ); + + let pli_res = + xtream_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target, None).await; + let mut pli = match pli_res { + Ok(pli) => pli, + Err(err) => match &resolved_target { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) => { + tuliprox_repository::target_stream_item::member_to_xtream_item(group, 0, &target) + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } => { + tuliprox_repository::target_stream_item::member_to_xtream_item(group, *index, &target) + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Item => { + error!("Failed to read xtream item for stream id {req_virtual_id}: {err}"); + if is_hls_manifest_request { + return hls_custom_video_manifest_response( + app_state, + &user, + CustomVideoStreamType::ChannelUnavailable, + axum::http::StatusCode::NOT_FOUND, + ) + .await; + } + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::ChannelUnavailable, + ) + .into_response(); + } + }, }; + let virtual_id = pli.virtual_id; let input_option = app_state.app_config.get_input_by_name(&pli.input_name); let stream_ext = input_option .as_deref() @@ -346,8 +372,12 @@ async fn xtream_player_api_stream( .into_response(); } - let virtual_id = pli.virtual_id; - if app_state.active_users.is_user_blocked_for_stream(&user.username, virtual_id).await { + let check_vid = match &resolved_target { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) + | tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, .. } => group.group_vid, + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Item => virtual_id, + }; + if app_state.active_users.is_user_blocked_for_stream(&user.username, check_vid).await { return axum::http::StatusCode::BAD_REQUEST.into_response(); } @@ -441,23 +471,25 @@ async fn xtream_player_api_stream( (pli.xtream_cluster, pli.item_type) }; - pli.url = match resolve_initial_stalker_playback_url( - app_state, - &input, - &user, - pli.provider_id, - pli.xtream_cluster, - item_type, - &pli.url, - ) - .await - { - Ok(url) => url, - Err(err) => { - error!("Failed to resolve initial Stalker playback URL: {}", sanitize_sensitive_info(&err.to_string())); - return axum::http::StatusCode::BAD_GATEWAY.into_response(); - } - }; + if !is_group { + pli.url = match resolve_initial_stalker_playback_url( + app_state, + &input, + &user, + pli.provider_id, + pli.xtream_cluster, + item_type, + &pli.url, + ) + .await + { + Ok(url) => url, + Err(err) => { + error!("Failed to resolve initial Stalker playback URL: {}", sanitize_sensitive_info(&err.to_string())); + return axum::http::StatusCode::BAD_GATEWAY.into_response(); + } + }; + } let requested_extension = if m3u_timeshift.is_some() { extract_extension_from_url(&pli.url).map(|ext| concat_string!(".", ext)) @@ -555,6 +587,8 @@ async fn xtream_player_api_stream( if session.virtual_id == virtual_id.get() && is_seekable_media_request(cluster, req_headers, Some(playback_ext)) { // partial request means we are in reverse proxy mode, seek happened + let (input, stream_channel) = + crate::api::api_utils::sticky_playback_context(app_state, session, &input, stream_channel); return force_provider_stream_response( fingerprint, app_state, @@ -715,16 +749,40 @@ async fn xtream_player_api_stream( let pinned_provider = user_session.as_ref().filter(|_| item_type.requires_provider_affinity()).map(|session| &session.provider); + let source = match resolved_target { + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Group(group) => { + let bound_prefer = user_session + .as_ref() + .and_then(|session| session.member_binding.as_ref()) + .filter(|b| b.group_vid == group.group_vid.get()) + .map(|b| b.member_index); + PlaybackSource::Group { + group, + mode: tuliprox_session::channel_group::SelectionMode::Live, + prefer: bound_prefer, + } + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Member { group, index } => { + PlaybackSource::Group { + group, + mode: tuliprox_session::channel_group::SelectionMode::Live, + prefer: Some(index), + } + } + tuliprox_repository::target_stream_item::ResolvedStreamTarget::Item => { + PlaybackSource::Single { input: &input, stream_url: &stream_url } + } + }; + stream_response( fingerprint, app_state, session_key.as_str(), Some(request_class), stream_channel, - &stream_url, + source, pinned_provider, req_headers, - &input, &target, &user, connection_permission, @@ -1077,10 +1135,9 @@ pub(in crate::api) async fn xtream_player_api_stream_with_resolved_target( session_key.as_str(), None, pli.to_stream_channel(target.id), - &stream_url, + PlaybackSource::Single { input: &input, stream_url: &stream_url }, None, req_headers, - &input, &target, &user, UserConnectionPermission::Allowed, diff --git a/backend/app/src/api/endpoints/xtream_api/tests.rs b/backend/app/src/api/endpoints/xtream_api/tests.rs index 09e43d1ca..9422ee5fb 100644 --- a/backend/app/src/api/endpoints/xtream_api/tests.rs +++ b/backend/app/src/api/endpoints/xtream_api/tests.rs @@ -912,6 +912,7 @@ async fn occupy_the_only_connection(app_state: &Arc, user: &ProxyUserC let channel = shared::model::StreamChannel { target_id: 1, virtual_id: 7, + group_virtual_id: None, provider_id: 7, input_name: "provider-b".intern(), item_type: PlaylistItemType::Live, diff --git a/backend/app/src/api/main_api.rs b/backend/app/src/api/main_api.rs index 3a698103d..86263e400 100644 --- a/backend/app/src/api/main_api.rs +++ b/backend/app/src/api/main_api.rs @@ -19,12 +19,12 @@ use crate::{ model::{ create_cache, create_http_client, create_http_client_no_redirect, create_public_http_client_no_redirect, create_resource_http_client_no_redirect, create_resource_public_http_client_no_redirect, exec_provider_dns, - exec_qos_aggregation, load_playlists_into_memory_cache, + exec_qos_aggregation_with_listener, load_playlists_into_memory_cache, recording_rule_scheduler::spawn_recording_rule_scheduler, - recording_supervisor::start_recording_supervisors, ActiveProviderManager, ActiveUserManager, AppState, - AuthState, CancelTokens, ConnectionManager, EventManager, EventMessage, HdHomerunAppState, HlsState, - ManualPlaylistUpdateRequest, MetadataUpdateManager, PlaylistStorageState, PlaylistUpdateControl, - RecordingQueue, SharedStreamManager, + recording_supervisor::start_recording_supervisors, ActiveProviderManager, ActiveUserManager, + AppQosAggregationListener, AppState, AuthState, CancelTokens, ConnectionManager, EventManager, + EventMessage, HdHomerunAppState, HlsState, ManualPlaylistUpdateRequest, MetadataUpdateManager, + PlaylistStorageState, PlaylistUpdateControl, RecordingQueue, SharedStreamManager, }, panel_api::{sync_panel_api_exp_dates, sync_panel_api_exp_dates_on_boot}, serve::serve, @@ -291,6 +291,49 @@ pub(crate) async fn load_maintenance(app_config: &AppConfig) -> Arc) -> Result, TuliproxError> { + Ok(Arc::new(HttpClients::new( + create_http_client(app_config)?, + create_http_client_no_redirect(app_config)?, + create_public_http_client_no_redirect(app_config)?, + create_resource_http_client_no_redirect(app_config)?, + create_resource_public_http_client_no_redirect(app_config)?, + ))) +} + +/// Populate initial `QoS` snapshots from repository into the health index. +fn populate_initial_member_health(storage_dir: &std::path::Path, member_health: &tuliprox_session::MemberHealthIndex) { + let mut initial_snapshots = Vec::new(); + let _ = tuliprox_repository::QosSnapshotRepository::for_each_snapshot_read_only(storage_dir, |record| { + initial_snapshots.push(record.clone()); + }); + if initial_snapshots.is_empty() { + return; + } + let entries = initial_snapshots.iter().map(|record| { + let key = if record.provider_id > 0 { + tuliprox_session::QosKey::Provider { + input: Arc::clone(&record.input_name), + provider_id: record.provider_id, + } + } else { + tuliprox_session::QosKey::TargetMember { + target: Arc::clone(&record.target_name), + member_vid: record.virtual_id, + } + }; + let part = tuliprox_session::QosPart { + score_24h: record.window_24h.score, + score_7d: record.window_7d.score, + confidence: record.window_24h.confidence, + failure_streak: record.window_24h.successive_failure_streak, + }; + (key, part) + }); + member_health.replace_qos(entries); +} + async fn create_shared_data( app_config: &Arc, forced_targets: &Arc, @@ -352,11 +395,7 @@ async fn create_shared_data( cleanup_capacity, )); - let client = create_http_client(app_config)?; - let client_no_redirect = create_http_client_no_redirect(app_config)?; - let public_client_no_redirect = create_public_http_client_no_redirect(app_config)?; - let resource_client_no_redirect = create_resource_http_client_no_redirect(app_config)?; - let resource_public_client_no_redirect = create_resource_public_http_client_no_redirect(app_config)?; + let http_clients = create_all_http_clients(app_config)?; let tokens = CancelTokens::default(); let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); @@ -372,16 +411,14 @@ async fn create_shared_data( })?; let maintenance = load_maintenance(app_config).await; + let member_health = Arc::new(tuliprox_session::MemberHealthIndex::default()); + event_manager.bind_member_health(&member_health); + member_health.set_event_sink(Arc::clone(&event_manager) as Arc); + populate_initial_member_health(std::path::Path::new(&config.storage_dir), &member_health); let app_state = AppState { app_config: Arc::clone(app_config), - http_clients: Arc::new(HttpClients::new( - client, - client_no_redirect, - public_client_no_redirect, - resource_client_no_redirect, - resource_public_client_no_redirect, - )), + http_clients, recordings: Arc::new(recordings), cache: Arc::new(ArcSwapOption::from(cache)), shared_stream_manager, @@ -403,6 +440,7 @@ async fn create_shared_data( auth: AuthState::new(token_revocations), playlist_updates: PlaylistUpdateControl::new(Arc::clone(forced_targets), manual_update_sender), maintenance, + member_health, }; Ok((app_state, manual_update_rx)) @@ -709,7 +747,11 @@ pub async fn start_server(app_config: Arc, targets: Arc, + pub member_health: Arc, } impl AppState { @@ -121,3 +122,33 @@ pub struct HdHomerunAppState { pub device: Arc, pub hd_scan_state: Arc, } + +pub struct AppQosAggregationListener { + pub member_health: Arc, +} + +impl tuliprox_session::qos_aggregation_manager::QosAggregationListener for AppQosAggregationListener { + fn on_snapshots(&self, snapshots: &[tuliprox_repository::QosSnapshotRecord]) { + let entries = snapshots.iter().map(|record| { + let key = if record.provider_id > 0 { + tuliprox_session::QosKey::Provider { + input: Arc::clone(&record.input_name), + provider_id: record.provider_id, + } + } else { + tuliprox_session::QosKey::TargetMember { + target: Arc::clone(&record.target_name), + member_vid: record.virtual_id, + } + }; + let part = tuliprox_session::QosPart { + score_24h: record.window_24h.score, + score_7d: record.window_7d.score, + confidence: record.window_24h.confidence, + failure_streak: record.window_24h.successive_failure_streak, + }; + (key, part) + }); + self.member_health.replace_qos(entries); + } +} diff --git a/backend/app/src/api/model/app_state/reload.rs b/backend/app/src/api/model/app_state/reload.rs index 400366f6b..ea3c32dc6 100644 --- a/backend/app/src/api/model/app_state/reload.rs +++ b/backend/app/src/api/model/app_state/reload.rs @@ -17,7 +17,9 @@ use shared::{error::TuliproxError, model::WebAuthConfigDto}; use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio_util::sync::CancellationToken; use tuliprox_dvr::recording::recording_transfer::{resume_recording_worker_if_needed, spawn_recording_services}; -use tuliprox_session::{provider_dns_manager::exec_provider_dns, qos_aggregation_manager::exec_qos_aggregation}; +use tuliprox_session::{ + provider_dns_manager::exec_provider_dns, qos_aggregation_manager::exec_qos_aggregation_with_listener, +}; macro_rules! cancel_service { ($field: ident, $flag:expr, $changes:expr, $cancel_tokens:expr) => { @@ -166,7 +168,11 @@ fn start_services(app_state: &Arc, changes: &UpdateChanges) { exec_provider_dns(&app_state.app_config, &app_state.cancel_tokens.load().provider_dns); } if changes.flags.contains(UpdateChangesFlags::QosAggregation) { - exec_qos_aggregation(&app_state.app_config, &app_state.cancel_tokens.load().qos_aggregation); + exec_qos_aggregation_with_listener( + &app_state.app_config, + &app_state.cancel_tokens.load().qos_aggregation, + Some(Arc::new(super::AppQosAggregationListener { member_health: Arc::clone(&app_state.member_health) })), + ); let history_cfg = app_state.app_config.config.load().reverse_proxy.as_ref().and_then(|rp| rp.stream_history.clone()); let connection_manager = Arc::clone(&app_state.connection_manager); diff --git a/backend/app/src/api/model/app_state/test_support.rs b/backend/app/src/api/model/app_state/test_support.rs index 5dbf3e16a..1b12a3205 100644 --- a/backend/app/src/api/model/app_state/test_support.rs +++ b/backend/app/src/api/model/app_state/test_support.rs @@ -88,6 +88,7 @@ pub fn create_test_app_state(config: Config) -> Arc { auth: AuthState::for_tests(), playlist_updates: PlaylistUpdateControl::for_tests(), maintenance, + member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()), }) } diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs b/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs index 5a71afef0..74bbf4169 100644 --- a/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs +++ b/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs @@ -90,6 +90,7 @@ pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_a auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), maintenance: crate::api::model::create_test_maintenance(""), + member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()), }) } diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/support.rs b/backend/app/src/api/model/streams/active_client_stream/tests/support.rs index b85a40a93..3679a8e44 100644 --- a/backend/app/src/api/model/streams/active_client_stream/tests/support.rs +++ b/backend/app/src/api/model/streams/active_client_stream/tests/support.rs @@ -122,6 +122,7 @@ pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_a auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), maintenance: crate::api::model::create_test_maintenance(""), + member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()), }) } @@ -147,6 +148,7 @@ pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_s StreamChannel { target_id: 1, virtual_id, + group_virtual_id: None, provider_id: 1, input_name: "input".intern(), item_type: PlaylistItemType::Live, diff --git a/backend/app/src/api/model/streams/provider_stream_factory.rs b/backend/app/src/api/model/streams/provider_stream_factory.rs index 97be489b9..8226e1859 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory.rs @@ -44,7 +44,8 @@ create_bitset!( RangeRequested, PublicDestinationRequired, HlsResource, - FlussonicAudioTracks + FlussonicAudioTracks, + GroupMember ); #[derive(Debug, Clone)] diff --git a/backend/app/src/api/model/streams/provider_stream_factory/opening.rs b/backend/app/src/api/model/streams/provider_stream_factory/opening.rs index 703a28e4a..cfed6f566 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory/opening.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory/opening.rs @@ -124,7 +124,9 @@ pub async fn create_provider_stream( Some(status), Some(failure.provider_error_class()), ); - if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) { + if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) + || stream_options.flags.contains(ProviderStreamFactoryFlags::GroupMember) + { let headers = match failure { ProviderStreamRequestFailure::HlsStatus { headers, .. } => headers, ProviderStreamRequestFailure::Status { .. } => Vec::new(), diff --git a/backend/app/src/api/model/streams/provider_stream_factory/options.rs b/backend/app/src/api/model/streams/provider_stream_factory/options.rs index 86713f8d6..24924d757 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory/options.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory/options.rs @@ -42,6 +42,41 @@ pub(crate) struct ProviderStreamFactoryParams<'a> { pub content_representation: ProviderContentRepresentationMode, } +fn compute_flags( + stream_options: &StreamOptions, + item_type: PlaylistItemType, + share_stream: bool, + range_requested: bool, + is_group_member: bool, +) -> ProviderStreamFactoryFlagsSet { + let mut flags = ProviderStreamFactoryFlagsSet::new(); + if stream_options.stream_retry { + flags.set(ProviderStreamFactoryFlags::RetryEnabled); + if !item_type.is_live_adaptive() && stream_options.response_mode != StreamResponseMode::HlsResource { + flags.set(ProviderStreamFactoryFlags::InitialRetryLoopEnabled); + } + } + if stream_options.pipe_provider_stream { + flags.set(ProviderStreamFactoryFlags::PipeStream); + } + if stream_options.buffer_enabled { + flags.set(ProviderStreamFactoryFlags::BufferEnabled); + } + if stream_options.response_mode == StreamResponseMode::HlsResource { + flags.set(ProviderStreamFactoryFlags::HlsResource); + } + if share_stream { + flags.set(ProviderStreamFactoryFlags::ShareStream); + } + if range_requested { + flags.set(ProviderStreamFactoryFlags::RangeRequested); + } + if is_group_member { + flags.set(ProviderStreamFactoryFlags::GroupMember); + } + flags +} + impl ProviderStreamFactoryOptions { pub(crate) fn new(request: &ProviderStreamFactoryParams<'_>) -> Self { let ProviderStreamFactoryParams { @@ -97,28 +132,13 @@ impl ProviderStreamFactoryOptions { }) .and_then(|ua| axum::http::header::HeaderValue::from_str(ua).ok()); let url = (*stream_url).clone(); - let mut flags = ProviderStreamFactoryFlagsSet::new(); - if stream_options.stream_retry { - flags.set(ProviderStreamFactoryFlags::RetryEnabled); - if !item_type.is_live_adaptive() && stream_options.response_mode != StreamResponseMode::HlsResource { - flags.set(ProviderStreamFactoryFlags::InitialRetryLoopEnabled); - } - } - if stream_options.pipe_provider_stream { - flags.set(ProviderStreamFactoryFlags::PipeStream); - } - if stream_options.buffer_enabled { - flags.set(ProviderStreamFactoryFlags::BufferEnabled); - } - if stream_options.response_mode == StreamResponseMode::HlsResource { - flags.set(ProviderStreamFactoryFlags::HlsResource); - } - if *share_stream { - flags.set(ProviderStreamFactoryFlags::ShareStream); - } - if requested_range.is_some() { - flags.set(ProviderStreamFactoryFlags::RangeRequested); - } + let flags = compute_flags( + stream_options, + *item_type, + *share_stream, + requested_range.is_some(), + stream_channel.as_ref().and_then(|ch| ch.group_virtual_id).is_some(), + ); Self { item_type: *item_type, diff --git a/backend/app/src/api/model/streams/provider_stream_factory/request.rs b/backend/app/src/api/model/streams/provider_stream_factory/request.rs index fa58c5281..e7467b356 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory/request.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory/request.rs @@ -183,8 +183,10 @@ pub(super) async fn send_with_manual_redirects( provider.as_ref(), true, stream_options.should_retry_provider_request(), - RequestFetchOptions::default() - .with_http_error_responses(stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource)), + RequestFetchOptions::default().with_http_error_responses( + stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) + || stream_options.flags.contains(ProviderStreamFactoryFlags::GroupMember), + ), |resolved_url| prepare_client(request_client, stream_options, Some(resolved_url), credential_state).0, ) .await; @@ -263,8 +265,10 @@ async fn provider_stream_request( provider.as_ref(), false, stream_options.should_retry_provider_request(), - RequestFetchOptions::default() - .with_http_error_responses(stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource)), + RequestFetchOptions::default().with_http_error_responses( + stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) + || stream_options.flags.contains(ProviderStreamFactoryFlags::GroupMember), + ), |resolved_url| { let (client, _partial_content) = prepare_client( request_client, @@ -436,6 +440,10 @@ pub(super) async fn get_provider_stream( debug_if_enabled!("stream provider {}", sanitize_sensitive_info(log_url.as_ref())); let start = Instant::now(); let mut connect_err: u32 = 1; + let is_group_member = stream_options.flags.contains(ProviderStreamFactoryFlags::GroupMember); + let max_retry_count = if is_group_member { 1 } else { ERR_MAX_RETRY_COUNT }; + let retry_seconds = if is_group_member { RETRY_SECONDS.min(3) } else { RETRY_SECONDS }; + let mut last_failure: Option = None; let handle_cancel = stream_options.get_cancel_token(); while stream_options.should_continue() { @@ -464,7 +472,7 @@ pub(super) async fn get_provider_stream( return Ok(Some(stream_response)); } Ok(None) => { - if connect_err > ERR_MAX_RETRY_COUNT { + if connect_err >= max_retry_count { warn!( "The stream could be unavailable. {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref()) @@ -495,26 +503,29 @@ pub(super) async fn get_provider_stream( "The stream could be unavailable. ({status}) {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref()) ); + last_failure = Some(failure); break; } - if connect_err > ERR_MAX_RETRY_COUNT { + if connect_err >= max_retry_count { warn!( "The stream could be unavailable. ({status}) {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref()) ); + last_failure = Some(failure); break; } + last_failure = Some(failure); } } - if !stream_options.should_continue() || connect_err > ERR_MAX_RETRY_COUNT { + if !stream_options.should_continue() || connect_err >= max_retry_count { break; } if !stream_options.should_retry_initial_open_loop() { break; } - if start.elapsed().as_secs() > RETRY_SECONDS { + if start.elapsed().as_secs() > retry_seconds { warn!( - "The stream could be unavailable. Giving up after {RETRY_SECONDS} seconds. {}", + "The stream could be unavailable. Giving up after {retry_seconds} seconds. {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref()) ); break; @@ -533,6 +544,9 @@ pub(super) async fn get_provider_stream( } debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); stream_options.cancel_reconnect(); + if let Some(failure) = last_failure { + return Err(failure); + } Err(ProviderStreamRequestFailure::Status { status: StatusCode::SERVICE_UNAVAILABLE, provider_error_class: "service_unavailable", diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/streaming.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/streaming.rs index b453c5361..77afa62f6 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory/tests/streaming.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/streaming.rs @@ -102,6 +102,7 @@ fn test_provider_stream_factory_options_builds_connect_failed_stream_info_from_h stream_channel: Some(&StreamChannel { target_id: 1, virtual_id: 77, + group_virtual_id: None, provider_id: 3, input_name: "input-a".intern(), item_type: PlaylistItemType::Live, diff --git a/backend/app/src/api/serve.rs b/backend/app/src/api/serve.rs index c61d5c8e1..48c8093f3 100644 --- a/backend/app/src/api/serve.rs +++ b/backend/app/src/api/serve.rs @@ -293,6 +293,7 @@ mod tests { let stream_channel = StreamChannel { target_id: 1, virtual_id: 1, + group_virtual_id: None, provider_id: 1, input_name: "input".intern(), item_type: PlaylistItemType::Series, diff --git a/backend/app/src/api/tasks/notification_bridge.rs b/backend/app/src/api/tasks/notification_bridge.rs index 575d88c12..2880878b4 100644 --- a/backend/app/src/api/tasks/notification_bridge.rs +++ b/backend/app/src/api/tasks/notification_bridge.rs @@ -287,7 +287,10 @@ pub fn to_notification(message: &EventMessage) -> Option { // which is the single place that decision is made. EventMessage::PlaylistUpdateProgress(_) | EventMessage::SystemInfoUpdate(_) - | EventMessage::RecordingProgress => None, + | EventMessage::RecordingProgress + | EventMessage::ChannelGroupMemberSelected { .. } + | EventMessage::ChannelGroupExhausted { .. } + | EventMessage::ChannelGroupCircuitChanged { .. } => None, } } @@ -631,6 +634,18 @@ mod tests { auth_audit(AuthAuditOutcome::SignInFailed), auth_audit(AuthAuditOutcome::SignInThrottled), auth_audit(AuthAuditOutcome::PermissionDenied), + EventMessage::ChannelGroupMemberSelected { + target_id: 1, + group_vid: 10, + member_vid: 11, + tier: shared::model::channel_group::CapacityTier::Free, + }, + EventMessage::ChannelGroupExhausted { target_id: 1, group_vid: 10 }, + EventMessage::ChannelGroupCircuitChanged { + input_name: "test".into(), + input_stream_id: "1".into(), + state: shared::model::channel_group::CircuitState::Closed, + }, ]; assert_eq!(samples.len(), EventKind::ALL.len(), "add the new variant to this list"); samples diff --git a/backend/core/src/model/config/target.rs b/backend/core/src/model/config/target.rs index bf1b86976..34e4eedab 100644 --- a/backend/core/src/model/config/target.rs +++ b/backend/core/src/model/config/target.rs @@ -9,9 +9,9 @@ use shared::{ apply_flags, create_bitset, foundation::{Filter, ValueProvider}, model::{ - ConfigTargetDto, ConfigTargetOptions, DeduplicateConfig, HdHomeRunTargetOutputDto, M3uTargetOutputDto, - MappingStage, PlaylistItemType, ProcessingOrder, StrmExportStyle, StrmTargetOutputDto, TargetOutputDto, - TargetType, TraktConfigDto, WatchContext, WatchContextSet, XtreamTargetOutputDto, + ChannelGroupsConfig, ConfigTargetDto, ConfigTargetOptions, DeduplicateConfig, HdHomeRunTargetOutputDto, + M3uTargetOutputDto, MappingStage, PlaylistItemType, ProcessingOrder, StrmExportStyle, StrmTargetOutputDto, + TargetOutputDto, TargetType, TraktConfigDto, WatchContext, WatchContextSet, XtreamTargetOutputDto, }, }; use std::{borrow::Cow, sync::Arc}; @@ -31,6 +31,7 @@ pub struct TargetExecutionPlan { pub transform_stages: Vec, pub pre_transform_identity_dedup: bool, pub post_merge_content_dedup: Option, + pub channel_groups: Option, } impl Default for TargetExecutionPlan { @@ -39,6 +40,7 @@ impl Default for TargetExecutionPlan { transform_stages: vec![TransformStage::Filter, TransformStage::Rename, TransformStage::Map], pre_transform_identity_dedup: false, post_merge_content_dedup: None, + channel_groups: None, } } } @@ -57,6 +59,11 @@ impl TargetExecutionPlan { transform_stages, pre_transform_identity_dedup: dto.options.as_ref().is_some_and(|options| options.remove_duplicates), post_merge_content_dedup: dto.options.as_ref().and_then(|options| options.deduplicate), + channel_groups: dto + .options + .as_ref() + .and_then(|options| options.channel_groups.clone()) + .filter(|cfg| cfg.enabled), } } } diff --git a/backend/core/src/model/provider.rs b/backend/core/src/model/provider.rs index 175540064..6fff618b9 100644 --- a/backend/core/src/model/provider.rs +++ b/backend/core/src/model/provider.rs @@ -534,6 +534,30 @@ impl ProviderConfig { true } + pub fn probe_capacity(&self, grace: bool, grace_period_timeout_secs: u64) -> ProviderConfigAllocation { + let guard = self.read_connection(); + if self.is_account_blocked_with_connection(&guard) { + return ProviderConfigAllocation::Exhausted; + } + + if self.max_connections == 0 { + return ProviderConfigAllocation::Available; + } + let connections = guard.current_connections; + if connections < self.max_connections { + return ProviderConfigAllocation::Available; + } + if !grace || connections > self.max_connections { + return ProviderConfigAllocation::Exhausted; + } + + let now = Instant::now(); + if Self::grace_is_active(&guard, now, grace_period_timeout_secs) { + return ProviderConfigAllocation::Exhausted; + } + ProviderConfigAllocation::GracePeriod + } + pub fn release(&self) { let mut guard = self.write_connection(); // Releasing while over capacity ends the single outstanding grace grant. @@ -575,6 +599,14 @@ impl ProviderConfigWrapper { } } + pub fn probe_capacity(&self, grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { + match self.inner.probe_capacity(grace, grace_period_timeout_secs) { + ProviderConfigAllocation::Available => ProviderAllocation::new_available(Arc::clone(&self.inner)), + ProviderConfigAllocation::GracePeriod => ProviderAllocation::new_grace_period(Arc::clone(&self.inner)), + ProviderConfigAllocation::Exhausted => ProviderAllocation::Exhausted, + } + } + pub fn get_next(&self, grace: bool, grace_period_timeout_secs: u64) -> Option> { if self.inner.get_next(grace, grace_period_timeout_secs) { return Some(Arc::clone(&self.inner)); diff --git a/backend/iptv/src/error.rs b/backend/iptv/src/error.rs index 1ae52ce45..9f84bec06 100644 --- a/backend/iptv/src/error.rs +++ b/backend/iptv/src/error.rs @@ -103,6 +103,7 @@ impl ProviderErrorKind { | ErrorKind::ConfigTarget | ErrorKind::ConfigWebUi | ErrorKind::TargetBouquet + | ErrorKind::ChannelGroup | ErrorKind::UrlParse | ErrorKind::FilterParse | ErrorKind::RegexCompile => Self::Config, diff --git a/backend/metadata/src/manager/task_processing.rs b/backend/metadata/src/manager/task_processing.rs index e6673d85f..7094b3ff8 100644 --- a/backend/metadata/src/manager/task_processing.rs +++ b/backend/metadata/src/manager/task_processing.rs @@ -538,6 +538,8 @@ impl InputWorker { Self::apply_pending_generic_probe_base(collector, cluster, provider_id, &mut item); + let raw_video = metadata.raw_video.clone(); + let raw_audio = metadata.raw_audio.clone(); update_properties( &mut item.additional_properties, *item_type, @@ -549,7 +551,17 @@ impl InputWorker { ); match item.additional_properties { - Some(StreamProperties::Live(props)) => collector.add_live(provider_id.into(), *props), + Some(StreamProperties::Live(props)) => { + let bitrate_kbps = props.bitrate / 1000; + ctx.events.observe_stream_probe( + Arc::clone(&input.name), + Arc::from(provider_id.to_string()), + raw_video.as_deref(), + raw_audio.as_deref(), + bitrate_kbps, + ); + collector.add_live(provider_id.into(), *props); + } Some(StreamProperties::Video(props)) => collector.add_vod(provider_id.into(), *props), _ => {} } diff --git a/backend/mpegts/Cargo.toml b/backend/mpegts/Cargo.toml index 82fd7c980..523b2b868 100644 --- a/backend/mpegts/Cargo.toml +++ b/backend/mpegts/Cargo.toml @@ -26,3 +26,7 @@ sha2.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["io-util"] } zeroize.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + diff --git a/backend/mpegts/src/lib.rs b/backend/mpegts/src/lib.rs index 1e7c9dbb1..3377189c8 100644 --- a/backend/mpegts/src/lib.rs +++ b/backend/mpegts/src/lib.rs @@ -12,3 +12,74 @@ pub mod transport_stream_buffer; pub mod ts_inspector; + +pub const TS_PACKET_SIZE: usize = 188; +pub const TS_SYNC_BYTE: u8 = 0x47; + +/// Finds the first offset `o < 188` where `0x47` appears at `o`, `o + 188`, `o + 376`. +/// With fewer than 564 bytes at EOF, all available positions must match. +#[must_use] +pub fn ts_sync_offset(buf: &[u8]) -> Option { + if buf.is_empty() { + return None; + } + let limit = TS_PACKET_SIZE.min(buf.len()); + 'offset_loop: for o in 0..limit { + let mut pos = o; + while pos < buf.len() && pos <= o + 2 * TS_PACKET_SIZE { + if buf[pos] != TS_SYNC_BYTE { + continue 'offset_loop; + } + pos += TS_PACKET_SIZE; + } + return Some(o); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ts_sync_offset_empty() { + assert_eq!(ts_sync_offset(&[]), None); + } + + #[test] + fn ts_sync_offset_aligned() { + let mut buf = vec![0u8; 564]; + buf[0] = 0x47; + buf[188] = 0x47; + buf[376] = 0x47; + assert_eq!(ts_sync_offset(&buf), Some(0)); + } + + #[test] + fn ts_sync_offset_with_garbage_prefix() { + let mut buf = vec![0u8; 17]; + let mut ts = vec![0u8; 564]; + ts[0] = 0x47; + ts[188] = 0x47; + ts[376] = 0x47; + buf.extend(ts); + assert_eq!(ts_sync_offset(&buf), Some(17)); + } + + #[test] + fn ts_sync_offset_short_buffer_at_eof() { + let mut buf = vec![0u8; 200]; + buf[10] = 0x47; + buf[198] = 0x47; + assert_eq!(ts_sync_offset(&buf), Some(10)); + } + + #[test] + fn ts_sync_offset_corrupt_packet() { + let mut buf = vec![0u8; 564]; + buf[0] = 0x47; + buf[188] = 0x48; // corrupt + buf[376] = 0x47; + assert_eq!(ts_sync_offset(&buf), None); + } +} diff --git a/backend/processing/src/processor/channel_group/assemble.rs b/backend/processing/src/processor/channel_group/assemble.rs new file mode 100644 index 000000000..79f32a9bc --- /dev/null +++ b/backend/processing/src/processor/channel_group/assemble.rs @@ -0,0 +1,543 @@ +use crate::processor::match_key::{extract_normalized_match_key, MatchKeyNormalizer}; +use indexmap::IndexMap; +use shared::{ + model::{ + ChannelGroupOverridesDto, ChannelGroupPolicy, ChannelGroupsConfig, ChannelMemberRefDto, EpgConflictMode, + LiveStreamProperties, PlaylistGroup, PlaylistItem, StreamProperties, UUIDType, XtreamCluster, + }, + utils::{hash_string, token_quality, Internable}, +}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tuliprox_repository::{ChannelGroupOrigin, PendingChannelGroup}; + +pub(in crate::processor) fn strip_quality_words(value: &str) -> String { + let words: Vec<&str> = value.split_whitespace().collect(); + let mut kept = Vec::with_capacity(words.len()); + for word in words { + let core = word.trim_matches(|c: char| !c.is_alphanumeric()); + if !core.is_empty() && token_quality(core).is_some() { + continue; + } + kept.push(word); + } + if kept.is_empty() { + value.to_string() + } else { + kept.join(" ") + } +} + +fn is_eligible(item: &PlaylistItem) -> bool { + item.header.xtream_cluster == XtreamCluster::Live && !item.header.item_type.is_local() +} + +fn item_member_ref(item: &PlaylistItem) -> Option { + if !is_eligible(item) { + return None; + } + let stream_id = item.header.get_input_stream_id()?; + if stream_id.is_empty() { + return None; + } + Some(ChannelMemberRefDto { input: item.header.input_name.to_string(), stream_id: stream_id.to_string() }) +} + +struct GroupCandidate { + key: String, + origin: ChannelGroupOrigin, + policy: ChannelGroupPolicy, + positions: Vec<(usize, usize)>, + orphaned: Vec, + epg_split: bool, + name_override: Option, + logo_override: Option, + epg_override: Option, + playlist_group_override: Option, +} + +struct AutoBucket { + positions: Vec<(usize, usize)>, + has_dsl_forced: bool, + epg_split: bool, +} + +struct GroupSlot { + id: u32, + title: Arc, + xtream_cluster: XtreamCluster, + had_channels: bool, + channels: Vec>, +} + +fn build_member_index(playlist: &[PlaylistGroup]) -> HashMap { + let mut member_pos_map = HashMap::new(); + for (group_idx, group) in playlist.iter().enumerate() { + if group.xtream_cluster != XtreamCluster::Live { + continue; + } + for (channel_idx, channel) in group.channels.iter().enumerate() { + if let Some(mref) = item_member_ref(channel) { + member_pos_map.entry(mref).or_insert((group_idx, channel_idx)); + } + } + } + member_pos_map +} + +fn resolve_manual_groups( + config: &ChannelGroupsConfig, + overrides: Option<&ChannelGroupOverridesDto>, + member_pos_map: &HashMap, + taken_positions: &mut HashSet<(usize, usize)>, + candidates: &mut Vec, +) { + let Some(overrides) = overrides else { + return; + }; + for mg in &overrides.groups { + let mut matched_positions = Vec::new(); + let mut orphaned = Vec::new(); + for mref in &mg.members { + if let Some(&pos) = member_pos_map.get(mref) { + if !taken_positions.contains(&pos) { + taken_positions.insert(pos); + matched_positions.push(pos); + continue; + } + } + log::warn!( + "Manual channel group '{}' member '{}:{}' not found or already taken", + mg.id, + mref.input, + mref.stream_id + ); + orphaned.push(mref.clone()); + } + + if matched_positions.len() < usize::from(config.min_members) { + continue; + } + + candidates.push(GroupCandidate { + key: mg.id.clone(), + origin: ChannelGroupOrigin::Manual, + policy: mg.policy.unwrap_or(config.policy_default), + positions: matched_positions, + orphaned, + epg_split: false, + name_override: Some(mg.name.clone()), + logo_override: mg.logo.clone(), + epg_override: mg.epg_channel_id.clone(), + playlist_group_override: mg.playlist_group.clone(), + }); + } +} + +fn collect_auto_buckets( + config: &ChannelGroupsConfig, + playlist: &[PlaylistGroup], + excluded_set: &HashSet<&ChannelMemberRefDto>, + taken_positions: &HashSet<(usize, usize)>, +) -> IndexMap { + let normalizer = MatchKeyNormalizer::new(&config.match_normalize, config.match_as_ascii); + let mut auto_buckets: IndexMap = IndexMap::new(); + + for (group_idx, group) in playlist.iter().enumerate() { + if group.xtream_cluster != XtreamCluster::Live { + continue; + } + for (channel_idx, channel) in group.channels.iter().enumerate() { + let pos = (group_idx, channel_idx); + if taken_positions.contains(&pos) { + continue; + } + let Some(mref) = item_member_ref(channel) else { + continue; + }; + if excluded_set.contains(&mref) { + continue; + } + + let mut dsl_forced = false; + let key = match channel.header.channel_group.as_deref() { + Some("") => continue, + Some(forced) => { + dsl_forced = true; + normalizer.key(forced) + } + None => extract_normalized_match_key( + &channel.header, + config.match_by, + config.match_as_ascii, + Some(&normalizer), + ), + }; + + if key.trim().is_empty() { + continue; + } + + let bucket = auto_buckets.entry(key).or_insert_with(|| AutoBucket { + positions: Vec::new(), + has_dsl_forced: false, + epg_split: false, + }); + bucket.positions.push(pos); + if dsl_forced { + bucket.has_dsl_forced = true; + } + } + } + auto_buckets +} + +fn split_epg_buckets( + playlist: &[PlaylistGroup], + auto_buckets: IndexMap, +) -> IndexMap { + let mut split_buckets: IndexMap = IndexMap::new(); + for (key, bucket) in auto_buckets { + if bucket.has_dsl_forced { + split_buckets.insert(key, bucket); + continue; + } + + let mut epg_groups: IndexMap> = IndexMap::new(); + for &pos in &bucket.positions { + let ch = &playlist[pos.0].channels[pos.1]; + if let Some(epg) = ch.header.epg_channel_id.as_deref() { + let trimmed = epg.trim().to_lowercase(); + if !trimmed.is_empty() { + epg_groups.entry(trimmed).or_default().push(pos); + } + } + } + + if epg_groups.len() >= 2 { + for (epg_id, sub_positions) in epg_groups { + let sub_key = format!("{key}|{epg_id}"); + split_buckets + .insert(sub_key, AutoBucket { positions: sub_positions, has_dsl_forced: false, epg_split: true }); + } + } else { + split_buckets.insert(key, bucket); + } + } + split_buckets +} + +fn apply_auto_override( + bucket: &mut AutoBucket, + ao: &shared::model::AutoGroupOverrideDto, + playlist: &[PlaylistGroup], + member_pos_map: &HashMap, + taken_positions: &HashSet<(usize, usize)>, + excluded_set: &HashSet<&ChannelMemberRefDto>, +) { + if !ao.remove.is_empty() { + bucket.positions.retain(|&pos| { + let ch = &playlist[pos.0].channels[pos.1]; + !ao.remove.iter().any(|rem| { + ch.header.input_name.as_ref() == rem.input.as_str() + && ch.header.get_input_stream_id().as_deref() == Some(rem.stream_id.as_str()) + }) + }); + } + + for add_ref in &ao.add { + if let Some(&pos) = member_pos_map.get(add_ref) { + if !taken_positions.contains(&pos) && !excluded_set.contains(add_ref) && !bucket.positions.contains(&pos) { + bucket.positions.push(pos); + } + } + } + + if !ao.order.is_empty() { + let mut ordered = Vec::with_capacity(bucket.positions.len()); + let mut used = HashSet::new(); + + for ord_ref in &ao.order { + if let Some(&pos) = bucket.positions.iter().find(|&&pos| { + let ch = &playlist[pos.0].channels[pos.1]; + ch.header.input_name.as_ref() == ord_ref.input.as_str() + && ch.header.get_input_stream_id().as_deref() == Some(ord_ref.stream_id.as_str()) + }) { + if used.insert(pos) { + ordered.push(pos); + } + } + } + for &pos in &bucket.positions { + if !used.contains(&pos) { + ordered.push(pos); + } + } + bucket.positions = ordered; + } +} + +fn build_group_metadata( + candidate: &GroupCandidate, + rep_header: &shared::model::PlaylistItemHeader, + slots: &[GroupSlot], + config: &ChannelGroupsConfig, + group_uuid: UUIDType, +) -> shared::model::PlaylistItemHeader { + let mut group_header = rep_header.clone(); + group_header.uuid = group_uuid; + group_header.channel_group = None; + group_header.channel_group_uuid = Some(group_uuid); + + if let Some(name) = &candidate.name_override { + group_header.name = name.as_str().intern(); + group_header.title = name.as_str().intern(); + } else if config.name_strip_quality { + group_header.name = strip_quality_words(&rep_header.name).intern(); + group_header.title = strip_quality_words(&rep_header.title).intern(); + } + + if let Some(logo) = &candidate.logo_override { + group_header.logo = logo.as_str().intern(); + } else if group_header.logo.is_empty() { + for &pos in &candidate.positions { + if let Some(item) = slots.get(pos.0).and_then(|g| g.channels.get(pos.1)).and_then(Option::as_ref) { + if !item.header.logo.is_empty() { + group_header.logo = item.header.logo.clone(); + break; + } + } + } + } + + if let Some(epg) = &candidate.epg_override { + group_header.epg_channel_id = Some(epg.as_str().intern()); + } else if group_header.epg_channel_id.as_deref().is_none_or(|s| s.trim().is_empty()) { + for &pos in &candidate.positions { + if let Some(item) = slots.get(pos.0).and_then(|g| g.channels.get(pos.1)).and_then(Option::as_ref) { + if let Some(epg) = &item.header.epg_channel_id { + if !epg.trim().is_empty() { + group_header.epg_channel_id = Some(epg.clone()); + break; + } + } + } + } + } + + apply_archive_metadata(&mut group_header, candidate, slots); + group_header +} + +fn apply_archive_metadata( + group_header: &mut shared::model::PlaylistItemHeader, + candidate: &GroupCandidate, + slots: &[GroupSlot], +) { + let mut has_archive = false; + let mut max_archive_duration: Option = None; + for &pos in &candidate.positions { + if let Some(item) = slots.get(pos.0).and_then(|g| g.channels.get(pos.1)).and_then(Option::as_ref) { + if let Some(StreamProperties::Live(live)) = &item.header.additional_properties { + if live.tv_archive.unwrap_or(0) > 0 { + has_archive = true; + if let Some(dur) = live.tv_archive_duration { + max_archive_duration = Some(max_archive_duration.map_or(dur, |m| m.max(dur))); + } + } + } + } + } + + if has_archive { + match &mut group_header.additional_properties { + Some(StreamProperties::Live(live)) => { + live.tv_archive = Some(1); + live.tv_archive_duration = max_archive_duration; + } + _ => { + group_header.additional_properties = Some(StreamProperties::Live(Box::new(LiveStreamProperties { + name: group_header.name.clone(), + category_id: group_header.category_id, + stream_id: 0, + stream_icon: group_header.logo.clone(), + direct_source: "".intern(), + custom_sid: None, + added: None, + stream_type: None, + epg_channel_id: group_header.epg_channel_id.clone(), + tv_archive: Some(1), + tv_archive_duration: max_archive_duration, + is_adult: 0, + ..Default::default() + }))); + } + } + } +} + +fn collect_group_candidates( + config: &ChannelGroupsConfig, + overrides: Option<&ChannelGroupOverridesDto>, + playlist: &[PlaylistGroup], +) -> Vec { + let member_pos_map = build_member_index(playlist); + let mut taken_positions = HashSet::new(); + let mut candidates = Vec::new(); + + resolve_manual_groups(config, overrides, &member_pos_map, &mut taken_positions, &mut candidates); + + let excluded_set: HashSet<&ChannelMemberRefDto> = + overrides.map_or_else(HashSet::new, |o| o.excluded.iter().collect()); + + let mut auto_buckets = collect_auto_buckets(config, playlist, &excluded_set, &taken_positions); + + if config.epg_conflict == EpgConflictMode::Split { + auto_buckets = split_epg_buckets(playlist, auto_buckets); + } + + for (key, mut bucket) in auto_buckets { + let auto_override = overrides.and_then(|o| o.auto_overrides.iter().find(|ao| ao.auto_key == key)); + let origin = if auto_override.is_some() { ChannelGroupOrigin::AutoEdited } else { ChannelGroupOrigin::Auto }; + + if let Some(ao) = auto_override { + apply_auto_override(&mut bucket, ao, playlist, &member_pos_map, &taken_positions, &excluded_set); + } + + if bucket.positions.len() < usize::from(config.min_members) { + continue; + } + + for &pos in &bucket.positions { + taken_positions.insert(pos); + } + + candidates.push(GroupCandidate { + key: key.clone(), + origin, + policy: auto_override.and_then(|ao| ao.policy).unwrap_or(config.policy_default), + positions: bucket.positions, + orphaned: Vec::new(), + epg_split: bucket.epg_split, + name_override: auto_override.and_then(|ao| ao.name.clone()), + logo_override: auto_override.and_then(|ao| ao.logo.clone()), + epg_override: auto_override.and_then(|ao| ao.epg_channel_id.clone()), + playlist_group_override: auto_override.and_then(|ao| ao.playlist_group.clone()), + }); + } + + candidates +} + +fn rebuild_playlist_from_slots( + slots: Vec, + deferred_placements: Vec<(String, PlaylistItem)>, + playlist: &mut Vec, +) { + for slot in slots { + let channels: Vec = slot.channels.into_iter().flatten().collect(); + if channels.is_empty() && slot.had_channels { + continue; + } + playlist.push(PlaylistGroup { id: slot.id, title: slot.title, channels, xtream_cluster: slot.xtream_cluster }); + } + + for (pg_override, mut item) in deferred_placements { + if let Some(g) = playlist.iter_mut().find(|g| g.title.as_ref() == pg_override.as_str()) { + item.header.category_id = g.id; + item.header.group = g.title.clone(); + g.channels.push(item); + } else { + let next_id = playlist.iter().map(|g| g.id).max().unwrap_or(0) + 1; + item.header.category_id = next_id; + item.header.group = pg_override.as_str().intern(); + playlist.push(PlaylistGroup { + id: next_id, + title: pg_override.as_str().intern(), + channels: vec![item], + xtream_cluster: XtreamCluster::Live, + }); + } + } +} + +fn materialize_groups( + candidates: &[GroupCandidate], + config: &ChannelGroupsConfig, + target_name: &str, + mut slots: Vec, + playlist: &mut Vec, +) -> Vec { + let mut deferred_placements = Vec::new(); + let mut pending_groups = Vec::with_capacity(candidates.len()); + + for candidate in candidates { + let rep_pos = candidate.positions[0]; + let Some(rep_item) = slots.get(rep_pos.0).and_then(|g| g.channels.get(rep_pos.1)).and_then(Option::as_ref) + else { + continue; + }; + + let group_uuid = hash_string(&format!("channel-group|{target_name}|{}", candidate.key)); + let group_header = build_group_metadata(candidate, &rep_item.header, &slots, config, group_uuid); + + let mut members = Vec::with_capacity(candidate.positions.len()); + for &pos in &candidate.positions { + if let Some(item) = slots.get_mut(pos.0).and_then(|g| g.channels.get_mut(pos.1)).and_then(Option::take) { + members.push(item); + } + } + + let group_item = PlaylistItem { header: group_header }; + + if let Some(pg_override) = &candidate.playlist_group_override { + deferred_placements.push((pg_override.clone(), group_item)); + } else if let Some(slot) = slots.get_mut(rep_pos.0).and_then(|g| g.channels.get_mut(rep_pos.1)) { + *slot = Some(group_item); + } + + pending_groups.push(PendingChannelGroup { + group_uuid, + key: candidate.key.as_str().intern(), + origin: candidate.origin, + policy: candidate.policy, + members, + orphaned: candidate.orphaned.clone(), + epg_split: candidate.epg_split, + }); + } + + rebuild_playlist_from_slots(slots, deferred_placements, playlist); + pending_groups +} + +pub(in crate::processor) fn group_playlist( + config: &ChannelGroupsConfig, + overrides: Option<&ChannelGroupOverridesDto>, + target_name: &str, + playlist: &mut Vec, +) -> Vec { + if !config.enabled || config.min_members < 2 || playlist.is_empty() { + return Vec::new(); + } + + let candidates = collect_group_candidates(config, overrides, playlist); + if candidates.is_empty() { + return Vec::new(); + } + + let slots: Vec = playlist + .drain(..) + .map(|g| GroupSlot { + id: g.id, + title: g.title, + xtream_cluster: g.xtream_cluster, + had_channels: !g.channels.is_empty(), + channels: g.channels.into_iter().map(Some).collect(), + }) + .collect(); + + materialize_groups(&candidates, config, target_name, slots, playlist) +} diff --git a/backend/processing/src/processor/channel_group/mod.rs b/backend/processing/src/processor/channel_group/mod.rs new file mode 100644 index 000000000..c98f27afa --- /dev/null +++ b/backend/processing/src/processor/channel_group/mod.rs @@ -0,0 +1,5 @@ +mod assemble; +#[cfg(test)] +mod tests; + +pub(in crate::processor) use assemble::*; diff --git a/backend/processing/src/processor/channel_group/tests.rs b/backend/processing/src/processor/channel_group/tests.rs new file mode 100644 index 000000000..c167e2aa3 --- /dev/null +++ b/backend/processing/src/processor/channel_group/tests.rs @@ -0,0 +1,276 @@ +use super::*; +use shared::{ + model::{ + AutoGroupOverrideDto, ChannelGroupOverridesDto, ChannelGroupsConfig, ChannelMemberRefDto, EpgConflictMode, + LiveStreamProperties, ManualChannelGroupDto, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + StreamProperties, XtreamCluster, + }, + utils::Internable, +}; +use tuliprox_repository::ChannelGroupOrigin; + +fn live(input: &str, stream_id: &str, title: &str) -> PlaylistItem { + let mut header = PlaylistItemHeader { + input_name: input.intern(), + id: stream_id.intern(), + title: title.intern(), + name: title.intern(), + url: format!("http://{input}.invalid/live/u/p/{stream_id}.ts").intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }; + header.freeze_input_stream_id(); + header.gen_uuid(); + PlaylistItem { header } +} + +fn group(title: &str, channels: Vec) -> PlaylistGroup { + PlaylistGroup { id: 1, title: title.intern(), channels, xtream_cluster: XtreamCluster::Live } +} + +fn cfg() -> ChannelGroupsConfig { serde_json::from_str(r#"{"enabled":true}"#).unwrap() } + +fn with_epg(mut item: PlaylistItem, epg: &str) -> PlaylistItem { + item.header.epg_channel_id = Some(epg.intern()); + item +} + +fn overrides_with_manual(id: &str, members: &[(&str, &str)]) -> ChannelGroupOverridesDto { + let mut overrides = ChannelGroupOverridesDto::new("T"); + overrides.groups.push(ManualChannelGroupDto { + id: id.into(), + name: id.into(), + members: members + .iter() + .map(|(input, stream_id)| ChannelMemberRefDto { input: (*input).into(), stream_id: (*stream_id).into() }) + .collect(), + ..Default::default() + }); + overrides +} + +fn set_archive(item: &mut PlaylistItem, duration: i32) { + let props = LiveStreamProperties { tv_archive: Some(1), tv_archive_duration: Some(duration), ..Default::default() }; + item.header.additional_properties = Some(StreamProperties::Live(Box::new(props))); +} + +fn archive_of(item: &PlaylistItem) -> Option { + match &item.header.additional_properties { + Some(StreamProperties::Live(live)) => live.tv_archive_duration, + _ => None, + } +} + +#[test] +fn auto_groups_same_caption_across_inputs() { + let mut playlist = vec![group( + "News", + vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD"), live("solis", "2", "ARD")], + )]; + let pending = group_playlist(&cfg(), None, "T", &mut playlist); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].members.len(), 2); + assert_eq!(pending[0].origin, ChannelGroupOrigin::Auto); + let titles: Vec<_> = playlist[0].channels.iter().map(|c| c.header.title.to_string()).collect(); + assert_eq!(titles, vec!["ZDF HD", "ARD"]); // group item at representative position + assert_eq!(playlist[0].channels[0].header.uuid, pending[0].group_uuid); +} + +#[test] +fn single_member_key_is_not_grouped() { + let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("solis", "2", "ARD")])]; + assert!(group_playlist(&cfg(), None, "T", &mut playlist).is_empty()); + assert_eq!(playlist[0].channels.len(), 2); +} + +#[test] +fn dsl_opt_out_and_forced_key() { + let mut a = live("solis", "1", "ZDF HD"); + a.header.channel_group = Some("".intern()); + let mut b = live("korath", "7", "Zweites"); + b.header.channel_group = Some("zdf hd".intern()); + let c = live("veyra", "3", "ZDF"); + let mut playlist = vec![group("News", vec![a, b, c])]; + let pending = group_playlist(&cfg(), None, "T", &mut playlist); + assert_eq!(pending.len(), 1); + let inputs: Vec<_> = pending[0].members.iter().map(|m| m.header.input_name.to_string()).collect(); + assert_eq!(inputs, vec!["korath", "veyra"]); +} + +#[test] +fn manual_group_wins_over_auto_and_keeps_order() { + let overrides = overrides_with_manual("zdf", &[("korath", "7"), ("solis", "1")]); + let mut playlist = vec![group( + "News", + vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD"), live("veyra", "3", "ZDF")], + )]; + let pending = group_playlist(&cfg(), Some(&overrides), "T", &mut playlist); + let manual = pending.iter().find(|p| p.origin == ChannelGroupOrigin::Manual).expect("manual"); + let inputs: Vec<_> = manual.members.iter().map(|m| m.header.input_name.to_string()).collect(); + assert_eq!(inputs, vec!["korath", "solis"]); + // veyra alone is below min_members and stays a normal channel + assert!(playlist[0].channels.iter().any(|c| c.header.input_name.as_ref() == "veyra")); +} + +#[test] +fn excluded_member_never_grouped() { + let mut overrides = ChannelGroupOverridesDto::new("T"); + overrides.excluded.push(ChannelMemberRefDto { input: "solis".into(), stream_id: "1".into() }); + let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])]; + assert!(group_playlist(&cfg(), Some(&overrides), "T", &mut playlist).is_empty()); +} + +#[test] +fn auto_override_orders_and_marks_edited() { + let mut overrides = ChannelGroupOverridesDto::new("T"); + overrides.auto_overrides.push(AutoGroupOverrideDto { + auto_key: "zdf".into(), + order: vec![ChannelMemberRefDto { input: "korath".into(), stream_id: "7".into() }], + ..Default::default() + }); + let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])]; + let pending = group_playlist(&cfg(), Some(&overrides), "T", &mut playlist); + assert_eq!(pending[0].origin, ChannelGroupOrigin::AutoEdited); + assert_eq!(pending[0].members[0].header.input_name.as_ref(), "korath"); + assert_eq!(playlist[0].channels[0].header.title.as_ref(), "ZDF FHD"); // representative +} + +#[test] +fn orphaned_manual_member_is_skipped() { + let overrides = overrides_with_manual("zdf", &[("solis", "1"), ("korath", "7"), ("gone", "99")]); + let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])]; + let pending = group_playlist(&cfg(), Some(&overrides), "T", &mut playlist); + assert_eq!(pending[0].members.len(), 2); + assert_eq!(pending[0].orphaned, vec![ChannelMemberRefDto { input: "gone".into(), stream_id: "99".into() }]); +} + +#[test] +fn manual_group_below_min_members_dissolves() { + let overrides = overrides_with_manual("zdf", &[("solis", "1"), ("gone", "99")]); + let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("veyra", "2", "ARD")])]; + assert!(group_playlist(&cfg(), Some(&overrides), "T", &mut playlist).is_empty()); + assert_eq!(playlist[0].channels.len(), 2); +} + +#[test] +fn archive_is_max_of_members() { + let mut a = live("solis", "1", "ZDF HD"); + set_archive(&mut a, 3); + let mut b = live("korath", "7", "ZDF FHD"); + set_archive(&mut b, 7); + let mut playlist = vec![group("News", vec![a, b])]; + group_playlist(&cfg(), None, "T", &mut playlist); + assert_eq!(archive_of(&playlist[0].channels[0]), Some(7)); +} + +#[test] +fn vod_and_series_are_never_grouped() { + let mut a = live("solis", "1", "Film"); + a.header.item_type = PlaylistItemType::Video; + a.header.xtream_cluster = XtreamCluster::Video; + let mut b = live("korath", "2", "Film"); + b.header.item_type = PlaylistItemType::Video; + b.header.xtream_cluster = XtreamCluster::Video; + let mut playlist = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: vec![a, b], + xtream_cluster: XtreamCluster::Video, + }]; + assert!(group_playlist(&cfg(), None, "T", &mut playlist).is_empty()); +} + +#[test] +fn different_epg_ids_split_the_auto_bucket() { + let mut playlist = vec![group( + "Sport", + vec![ + with_epg(live("solis", "1", "Sport 1 HD"), "sport1.de"), + with_epg(live("korath", "7", "Sport 1"), "sport1.de"), + with_epg(live("veyra", "3", "Sport 1 FHD"), "sport1.uk"), + with_epg(live("orun", "4", "Sport 1"), "sport1.uk"), + live("solis", "9", "Sport 1"), // no EPG id → stays a normal channel + ], + )]; + let pending = group_playlist(&cfg(), None, "T", &mut playlist); + assert_eq!(pending.len(), 2); + assert!(pending.iter().all(|p| p.epg_split && p.members.len() == 2)); + assert!(playlist[0] + .channels + .iter() + .any(|c| c.header.input_name.as_ref() == "solis" && c.header.id.as_ref() == "9")); +} + +#[test] +fn epg_conflict_ignore_keeps_one_group() { + let mut config = cfg(); + config.epg_conflict = EpgConflictMode::Ignore; + let mut playlist = vec![group( + "Sport", + vec![ + with_epg(live("solis", "1", "Sport 1"), "sport1.de"), + with_epg(live("korath", "7", "Sport 1"), "sport1.uk"), + ], + )]; + let pending = group_playlist(&config, None, "T", &mut playlist); + assert_eq!((pending.len(), pending[0].epg_split), (1, false)); +} + +#[test] +fn dsl_forced_key_is_never_split() { + let mut a = with_epg(live("solis", "1", "Eins"), "a.de"); + a.header.channel_group = Some("eins".intern()); + let mut b = with_epg(live("korath", "7", "One"), "b.uk"); + b.header.channel_group = Some("eins".intern()); + let mut playlist = vec![group("News", vec![a, b])]; + let pending = group_playlist(&cfg(), None, "T", &mut playlist); + assert_eq!((pending.len(), pending[0].members.len()), (1, 2)); +} + +#[test] +fn match_normalize_groups_prefixed_and_unprefixed_captions() { + let mut config = cfg(); + config.match_normalize.strip_regex = Some(r"^\s*(?:\|[A-Z]{2,3}\||[A-Z]{2,3}\s*[:|])\s*".to_string()); + config.match_normalize.ignore_tokens = vec!["hevc".to_string()]; + config.prepare().expect("valid"); + let mut playlist = vec![group("News", vec![live("solis", "1", "DE: ZDF HEVC"), live("korath", "7", "ZDF")])]; + let pending = group_playlist(&config, None, "T", &mut playlist); + assert_eq!((pending.len(), pending[0].members.len()), (1, 2)); +} + +#[test] +fn name_strip_quality_cleans_derived_name_only() { + assert_eq!(strip_quality_words("ZDF [FHD]"), "ZDF"); + assert_eq!(strip_quality_words("DE: ZDF HD"), "DE: ZDF"); + assert_eq!(strip_quality_words("ZDF-HD"), "ZDF-HD"); + assert_eq!(strip_quality_words("HD"), "HD"); // nothing left → original + let mut config = cfg(); + config.name_strip_quality = true; + let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])]; + group_playlist(&config, None, "T", &mut playlist); + assert_eq!(playlist[0].channels[0].header.title.as_ref(), "ZDF"); +} + +#[test] +fn emptied_playlist_group_is_dropped_and_group_uuid_is_stable() { + let build = + || vec![group("A", vec![live("solis", "1", "ZDF HD")]), group("B", vec![live("korath", "7", "ZDF FHD")])]; + let mut first = build(); + let p1 = group_playlist(&cfg(), None, "T", &mut first); + let mut second = build(); + let p2 = group_playlist(&cfg(), None, "T", &mut second); + assert_eq!(first.len(), 1); + assert_eq!(p1[0].group_uuid, p2[0].group_uuid); +} + +#[test] +fn members_keep_original_uuid_and_group_item_is_marked() { + let a = live("solis", "1", "ZDF HD"); + let b = live("korath", "7", "ZDF FHD"); + let (uuid_a, uuid_b) = (a.header.uuid, b.header.uuid); + let mut playlist = vec![group("News", vec![a, b])]; + let pending = group_playlist(&cfg(), None, "T", &mut playlist); + assert_eq!(pending[0].members.iter().map(|m| m.header.uuid).collect::>(), vec![uuid_a, uuid_b]); + assert_eq!(playlist[0].channels[0].header.channel_group_uuid, Some(pending[0].group_uuid)); +} diff --git a/backend/processing/src/processor/deduplicate.rs b/backend/processing/src/processor/deduplicate.rs index 411dfd42d..59c48f414 100644 --- a/backend/processing/src/processor/deduplicate.rs +++ b/backend/processing/src/processor/deduplicate.rs @@ -1,45 +1,20 @@ +use super::match_key::{dedup_match_by, extract_normalized_match_key, raw_match_value}; use shared::{ - model::{DeduplicateConfig, DeduplicateKeep, DeduplicateMatchBy, PlaylistGroup, PlaylistItem, XtreamCluster}, - utils::{deunicode_string, quality_rank, quality_tokens, token_quality}, + model::{DeduplicateConfig, DeduplicateKeep, PlaylistGroup, PlaylistItem, UUIDType, XtreamCluster}, + utils::quality_rank, }; -use std::collections::HashMap; - -/// Lowercased token join with quality tokens removed, so "News HD" and -/// "NEWS [FHD]" produce the same key. -fn normalized_dedup_key(value: &str, match_as_ascii: bool) -> String { - let value = if match_as_ascii { deunicode_string(value) } else { std::borrow::Cow::Borrowed(value) }; - let mut key = String::with_capacity(value.len()); - for token in quality_tokens(&value) { - if token_quality(token).is_some() { - continue; - } - if !key.is_empty() { - key.push(' '); - } - for ch in token.chars() { - key.extend(ch.to_lowercase()); - } - } - key -} +use std::collections::{HashMap, HashSet}; fn match_value(config: DeduplicateConfig, item: &PlaylistItem) -> String { - normalized_dedup_key(raw_match_value(config, item), config.match_as_ascii) + extract_normalized_match_key(&item.header, dedup_match_by(config.match_by), config.match_as_ascii, None) } -fn raw_match_value(config: DeduplicateConfig, item: &PlaylistItem) -> &str { - let header = &item.header; - match config.match_by { - DeduplicateMatchBy::Caption => { - if header.title.is_empty() { - header.name.as_ref() - } else { - header.title.as_ref() - } - } - DeduplicateMatchBy::Name => header.name.as_ref(), - DeduplicateMatchBy::Title => header.title.as_ref(), - } +fn item_raw_match_value(config: DeduplicateConfig, item: &PlaylistItem) -> &str { + raw_match_value(&item.header, dedup_match_by(config.match_by)) +} + +fn should_skip(channel: &PlaylistItem, skip: &HashSet) -> bool { + skip.contains(&channel.header.uuid) || channel.header.channel_group_uuid.is_some() } /// Collapse duplicate channels across the whole target playlist (per cluster). @@ -48,16 +23,20 @@ fn raw_match_value(config: DeduplicateConfig, item: &PlaylistItem) -> &str { pub(in crate::processor) fn deduplicate_playlist( config: DeduplicateConfig, playlist: &mut Vec, + skip: &HashSet, ) -> usize { // winner per key: (quality rank, group index, channel index) let mut winners: HashMap<(XtreamCluster, String), (u8, usize, usize)> = HashMap::new(); for (group_idx, group) in playlist.iter().enumerate() { for (channel_idx, channel) in group.channels.iter().enumerate() { + if should_skip(channel, skip) { + continue; + } let key_value = match_value(config, channel); if key_value.is_empty() { continue; } - let rank = quality_rank(raw_match_value(config, channel)); + let rank = quality_rank(item_raw_match_value(config, channel)); match winners.entry((group.xtream_cluster, key_value)) { std::collections::hash_map::Entry::Vacant(entry) => { entry.insert((rank, group_idx, channel_idx)); @@ -81,6 +60,9 @@ pub(in crate::processor) fn deduplicate_playlist( group.channels.retain(|channel| { let idx = channel_idx; channel_idx += 1; + if should_skip(channel, skip) { + return true; + } let key_value = match_value(config, channel); if key_value.is_empty() { return true; @@ -104,7 +86,10 @@ pub(in crate::processor) fn deduplicate_playlist( #[cfg(test)] mod tests { use super::*; - use shared::{model::PlaylistItemHeader, utils::Internable}; + use shared::{ + model::{DeduplicateMatchBy, PlaylistItemHeader}, + utils::Internable, + }; fn make_item(title: &str) -> PlaylistItem { PlaylistItem { header: PlaylistItemHeader { title: title.intern(), ..Default::default() } } @@ -134,7 +119,7 @@ mod tests { keep: DeduplicateKeep::BestQuality, match_as_ascii: false, }; - let removed = deduplicate_playlist(config, &mut playlist); + let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new()); assert_eq!(removed, 2); let titles: Vec<_> = playlist[0].channels.iter().map(|c| c.header.title.to_string()).collect(); assert_eq!(titles, vec!["News [FHD]", "Sports HD"]); @@ -150,7 +135,7 @@ mod tests { match_as_ascii: false, }; - let removed = deduplicate_playlist(config, &mut playlist); + let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new()); // Target-wide curation evaluates only this surviving winner. assert_eq!(removed, 1); @@ -167,7 +152,7 @@ mod tests { keep: DeduplicateKeep::First, match_as_ascii: false, }; - let removed = deduplicate_playlist(config, &mut playlist); + let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new()); assert_eq!(removed, 1); assert_eq!(playlist[0].channels[0].header.title.as_ref(), "News HD"); } @@ -184,7 +169,7 @@ mod tests { keep: DeduplicateKeep::BestQuality, match_as_ascii: false, }; - let removed = deduplicate_playlist(config, &mut playlist); + let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new()); assert_eq!(removed, 1); // "A" was emptied by dedup and dropped, the already-empty group survives assert_eq!(playlist.len(), 2); @@ -203,7 +188,7 @@ mod tests { keep: DeduplicateKeep::BestQuality, match_as_ascii: true, }; - let removed = deduplicate_playlist(config, &mut playlist); + let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new()); assert_eq!(removed, 1); assert_eq!(playlist[0].channels[0].header.title.as_ref(), "Cafe FHD"); @@ -214,6 +199,17 @@ mod tests { keep: DeduplicateKeep::BestQuality, match_as_ascii: false, }; - assert_eq!(deduplicate_playlist(config, &mut playlist), 0); + assert_eq!(deduplicate_playlist(config, &mut playlist, &HashSet::new()), 0); + } + + #[test] + fn dedup_never_drops_group_item() { + let group_item = make_item("ZDF HD"); + let excluded = make_item("ZDF UHD"); + let skip: HashSet = [group_item.header.uuid].into_iter().collect(); + let mut playlist = vec![make_group("News", vec![group_item, excluded])]; + let removed = deduplicate_playlist(DeduplicateConfig::default(), &mut playlist, &skip); + assert_eq!(removed, 0); + assert_eq!(playlist[0].channels.len(), 2); } } diff --git a/backend/processing/src/processor/match_key.rs b/backend/processing/src/processor/match_key.rs new file mode 100644 index 000000000..3b9424e27 --- /dev/null +++ b/backend/processing/src/processor/match_key.rs @@ -0,0 +1,193 @@ +use regex::Regex; +use shared::{ + model::{ChannelMatchBy, ChannelMatchNormalize, DeduplicateMatchBy, PlaylistItemHeader}, + utils::{deunicode_string, quality_tokens, token_quality}, +}; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, +}; + +pub(in crate::processor) const fn dedup_match_by(value: DeduplicateMatchBy) -> ChannelMatchBy { + match value { + DeduplicateMatchBy::Caption => ChannelMatchBy::Caption, + DeduplicateMatchBy::Name => ChannelMatchBy::Name, + DeduplicateMatchBy::Title => ChannelMatchBy::Title, + } +} + +/// Lowercased token join with quality tokens removed, so "News HD" and +/// "NEWS [FHD]" produce the same key. +pub(in crate::processor) fn normalized_match_key(value: &str, match_as_ascii: bool) -> String { + let value = if match_as_ascii { deunicode_string(value) } else { std::borrow::Cow::Borrowed(value) }; + let mut key = String::with_capacity(value.len()); + for token in quality_tokens(&value) { + if token_quality(token).is_some() { + continue; + } + if !key.is_empty() { + key.push(' '); + } + for ch in token.chars() { + key.extend(ch.to_lowercase()); + } + } + key +} + +pub(in crate::processor) fn raw_match_value(header: &PlaylistItemHeader, match_by: ChannelMatchBy) -> &str { + match match_by { + ChannelMatchBy::Caption => { + if header.title.is_empty() { + header.name.as_ref() + } else { + header.title.as_ref() + } + } + ChannelMatchBy::Name => header.name.as_ref(), + ChannelMatchBy::Title => header.title.as_ref(), + ChannelMatchBy::EpgChannelId => header.epg_channel_id.as_deref().unwrap_or(""), + } +} + +pub(in crate::processor) fn extract_normalized_match_key( + header: &PlaylistItemHeader, + match_by: ChannelMatchBy, + match_as_ascii: bool, + normalizer: Option<&MatchKeyNormalizer<'_>>, +) -> String { + let raw = raw_match_value(header, match_by); + if match_by == ChannelMatchBy::EpgChannelId { + return raw.trim().to_lowercase(); + } + match normalizer { + Some(normalizer) => normalizer.key(raw), + None => normalized_match_key(raw, match_as_ascii), + } +} + +#[allow(dead_code)] +pub(in crate::processor) struct MatchKeyNormalizer<'a> { + strip: Option<&'a Regex>, + ignore: HashSet, + aliases: HashMap, + match_as_ascii: bool, +} + +#[allow(dead_code)] +impl<'a> MatchKeyNormalizer<'a> { + pub(in crate::processor) fn new(config: &'a ChannelMatchNormalize, match_as_ascii: bool) -> Self { + let mut normalizer = Self { + strip: config.t_strip_regex.as_deref(), + ignore: config.ignore_tokens.iter().map(|token| token.to_lowercase()).collect(), + aliases: HashMap::new(), + match_as_ascii, + }; + let aliases = + config.aliases.iter().map(|(from, to)| (normalizer.base_key(from), normalizer.base_key(to))).collect(); + normalizer.aliases = aliases; + normalizer + } + + pub(in crate::processor) fn key(&self, value: &str) -> String { + let key = self.base_key(value); + self.aliases.get(&key).cloned().unwrap_or(key) + } + + fn base_key(&self, value: &str) -> String { + let stripped = self.strip.map_or(Cow::Borrowed(value), |regex| regex.replace_all(value, "")); + let folded = if self.match_as_ascii { deunicode_string(&stripped) } else { stripped }; + let mut key = String::with_capacity(folded.len()); + for token in quality_tokens(&folded) { + let lower = token.to_lowercase(); + if token_quality(token).is_some() || self.ignore.contains(&lower) { + continue; + } + if !key.is_empty() { + key.push(' '); + } + key.push_str(&lower); + } + key + } +} + +#[cfg(test)] +mod tests { + use super::*; + use shared::{ + model::{ChannelGroupsConfig, ChannelMatchNormalize, PlaylistItemHeader}, + utils::Internable, + }; + + fn header(title: &str, name: &str, epg: Option<&str>) -> PlaylistItemHeader { + PlaylistItemHeader { + title: title.intern(), + name: name.intern(), + epg_channel_id: epg.map(Internable::intern), + ..Default::default() + } + } + + #[test] + fn strips_quality_tokens_and_lowercases() { + assert_eq!(normalized_match_key("News HD", false), "news"); + assert_eq!(normalized_match_key("NEWS [FHD]", false), normalized_match_key("news 1080p", false)); + } + + #[test] + fn caption_falls_back_to_name() { + assert_eq!( + extract_normalized_match_key(&header("", "Sport HD", None), ChannelMatchBy::Caption, false, None), + "sport" + ); + } + + #[test] + fn epg_channel_id_is_used_verbatim_lowercased() { + let h = header("ZDF HD", "ZDF", Some("ZDF.de")); + assert_eq!(extract_normalized_match_key(&h, ChannelMatchBy::EpgChannelId, false, None), "zdf.de"); + assert_eq!( + extract_normalized_match_key(&header("X", "X", None), ChannelMatchBy::EpgChannelId, false, None), + "" + ); + } + + #[test] + fn ascii_folding() { + assert_eq!(normalized_match_key("Café HD", true), normalized_match_key("Cafe FHD", true)); + } + + fn normalize(strip: Option<&str>, ignore: &[&str], aliases: &[(&str, &str)]) -> ChannelMatchNormalize { + let mut cfg: ChannelGroupsConfig = serde_json::from_str(r#"{"enabled":true}"#).expect("parses"); + cfg.match_normalize = ChannelMatchNormalize { + strip_regex: strip.map(str::to_string), + ignore_tokens: ignore.iter().map(|t| (*t).to_string()).collect(), + aliases: aliases.iter().map(|(k, v)| ((*k).to_string(), (*v).to_string())).collect(), + ..Default::default() + }; + cfg.prepare().expect("valid"); + cfg.match_normalize + } + + #[test] + fn normalizer_strips_prefix_ignores_tokens_and_resolves_aliases() { + let cfg = normalize( + Some(r"^\s*(?:\|[A-Z]{2,3}\||[A-Z]{2,3}\s*[:|])\s*"), + &["HEVC", "50fps"], + &[("Das Erste HD", "ARD")], + ); + let normalizer = MatchKeyNormalizer::new(&cfg, false); + assert_eq!(normalizer.key("DE: ZDF HEVC"), "zdf"); + assert_eq!(normalizer.key("|DE| ZDF 50FPS"), "zdf"); + assert_eq!(normalizer.key("ZDF"), "zdf"); + assert_eq!(normalizer.key("ZDF NEO"), "zdf neo"); + assert_eq!(normalizer.key("DE: Das Erste"), "ard"); + } + + #[test] + fn dedup_keys_unchanged_without_normalizer() { + let h = header("DE: ZDF HEVC", "", None); + assert_eq!(extract_normalized_match_key(&h, ChannelMatchBy::Caption, false, None), "de zdf hevc"); + } +} diff --git a/backend/processing/src/processor/mod.rs b/backend/processing/src/processor/mod.rs index 114f7a975..6d1f60cdc 100644 --- a/backend/processing/src/processor/mod.rs +++ b/backend/processing/src/processor/mod.rs @@ -3,9 +3,11 @@ mod stalker; mod stalker_refresh; mod xtream; // mod affix; +pub(in crate::processor) mod channel_group; mod deduplicate; mod epg; mod library; +pub(in crate::processor) mod match_key; mod probe_handle_guard; mod providers; mod resolve_options; diff --git a/backend/processing/src/processor/playlist/mod.rs b/backend/processing/src/processor/playlist/mod.rs index 7f0891f5d..05ad0b53d 100644 --- a/backend/processing/src/processor/playlist/mod.rs +++ b/backend/processing/src/processor/playlist/mod.rs @@ -59,8 +59,8 @@ use tuliprox_curation::{ SelectorOutcome, }; use tuliprox_repository::{ - persist_playlist_views, CategoryKey, MemoryPlaylistSource, PlaylistPublicationPlan, PlaylistSource, - PlaylistStorageState, TargetPlaylistPersistOptions, + persist_playlist_views_with_groups, CategoryKey, MemoryPlaylistSource, PendingChannelGroup, + PlaylistPublicationPlan, PlaylistSource, PlaylistStorageState, TargetPlaylistPersistOptions, }; use tuliprox_session::ActiveProviderManager; diff --git a/backend/processing/src/processor/playlist/target.rs b/backend/processing/src/processor/playlist/target.rs index 4e584f949..319688fa3 100644 --- a/backend/processing/src/processor/playlist/target.rs +++ b/backend/processing/src/processor/playlist/target.rs @@ -414,13 +414,14 @@ fn finalization_step_label(view: &str, stage: &str) -> String { pub(crate) fn prepare_eligible_catalog( target: &ConfigTarget, playlist: Vec, + group_uuids: &HashSet, step: &mut StepMeasure, ) -> Vec { let mut playlist = flatten_groups(playlist); step.tick("playlist merge"); log_memory_snapshot(format!("target '{}' after_playlist_merge", target.name).as_str()); if let Some(dedup_config) = target.execution_plan.post_merge_content_dedup.as_ref() { - let removed = crate::processor::deduplicate::deduplicate_playlist(*dedup_config, &mut playlist); + let removed = crate::processor::deduplicate::deduplicate_playlist(*dedup_config, &mut playlist, group_uuids); if removed > 0 { info!("Deduplicated {removed} eligible channels for target {}", target.name); } @@ -506,6 +507,28 @@ pub(crate) async fn finalize_prepared_target, + step: &mut StepMeasure, + errors: &mut Vec, +) -> Vec { + let Some(cg_cfg) = target.execution_plan.channel_groups.as_ref() else { + return Vec::new(); + }; + let overrides = + tuliprox_repository::load_channel_group_overrides(config, &target.name).await.unwrap_or_else(|err| { + errors.push(err); + None + }); + step.broadcast("Processing channel groups for '{}' playlist", &target.name); + let groups = crate::processor::channel_group::group_playlist(cg_cfg, overrides.as_ref(), &target.name, playlist); + step.tick("channel grouping"); + log_memory_snapshot(format!("target '{}' after_channel_grouping", target.name).as_str()); + groups +} + pub(super) async fn finalize_prepared_target_with_tmdb( ctx: Arc>, prepared: PreparedTarget, @@ -517,6 +540,10 @@ pub(super) async fn finalize_prepared_target_with_tmdb = pending_channel_groups.iter().map(|p| p.group_uuid).collect(); + let eligible_catalog = prepare_eligible_catalog(target, new_playlist, &group_uuids, &mut step); + let mut views = prepare_target_playlist_views(&ctx.client, tmdb_client, target, eligible_catalog).await; + views.channel_groups = pending_channel_groups; + if views.base.is_empty() && views.xtream.is_none() && views.publication_plan == PlaylistPublicationPlan::Ordinary @@ -577,7 +607,7 @@ pub(super) async fn finalize_prepared_target_with_tmdb, pub(crate) xtream: Option>, pub(crate) publication_plan: PlaylistPublicationPlan, + pub(crate) channel_groups: Vec, } pub(crate) fn build_curated_playlist_views( @@ -926,6 +958,7 @@ pub(crate) fn build_curated_playlist_views( has_xtream_output && !config.include_xtream_base_categories, appearance_filter_configured, ), + channel_groups: Vec::new(), } } @@ -940,6 +973,7 @@ pub(crate) async fn prepare_target_playlist_views( base: playlist, xtream: None, publication_plan: PlaylistPublicationPlan::Ordinary, + channel_groups: Vec::new(), }; }; @@ -954,13 +988,21 @@ pub(super) fn curation_playlist_views( outcome: CurationRunOutcome, ) -> TargetPlaylistViews { match outcome { - CurationRunOutcome::NotConfigured => { - TargetPlaylistViews { base: playlist, xtream: None, publication_plan: PlaylistPublicationPlan::Ordinary } - } + CurationRunOutcome::NotConfigured => TargetPlaylistViews { + base: playlist, + xtream: None, + publication_plan: PlaylistPublicationPlan::Ordinary, + channel_groups: Vec::new(), + }, CurationRunOutcome::Failed(failure) => { warn_curation_failure(&target.name, &failure); // Discovery is optional; unavailable selections must not suppress the source catalog or EPG. - TargetPlaylistViews { base: playlist, xtream: None, publication_plan: PlaylistPublicationPlan::Ordinary } + TargetPlaylistViews { + base: playlist, + xtream: None, + publication_plan: PlaylistPublicationPlan::Ordinary, + channel_groups: Vec::new(), + } } CurationRunOutcome::Complete(evaluation) => { let views = build_curated_playlist_views( @@ -1170,3 +1212,23 @@ pub(crate) async fn process_watch( true } + +#[cfg(test)] +pub(crate) async fn run_finalize_until_persist( + target: &ConfigTarget, + mut playlist: Vec, +) -> (Vec, Vec) { + let mut step = StepMeasure::new("test", |_, _| {}); + let pending = if let Some(cg_cfg) = target.execution_plan.channel_groups.as_ref() { + crate::processor::channel_group::group_playlist(cg_cfg, None, &target.name, &mut playlist) + } else { + Vec::new() + }; + if target.favourites.is_some() { + process_favourites(&mut playlist, target.favourites.as_deref()); + } + let group_uuids: HashSet = pending.iter().map(|p| p.group_uuid).collect(); + let eligible = prepare_eligible_catalog(target, playlist, &group_uuids, &mut step); + let finalized = finalize_playlist_view(target, eligible, &mut step, "base"); + (finalized, pending) +} diff --git a/backend/processing/src/processor/playlist/tests.rs b/backend/processing/src/processor/playlist/tests.rs index 5e97c9828..3c2452e3d 100644 --- a/backend/processing/src/processor/playlist/tests.rs +++ b/backend/processing/src/processor/playlist/tests.rs @@ -24,6 +24,8 @@ mod curation_effect_gate; mod watch_contexts; +mod channel_groups; + #[cfg(test)] mod quality_rejection_fallback; diff --git a/backend/processing/src/processor/playlist/tests/channel_groups.rs b/backend/processing/src/processor/playlist/tests/channel_groups.rs new file mode 100644 index 000000000..412e7acd6 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/channel_groups.rs @@ -0,0 +1,89 @@ +use crate::processor::playlist::target::run_finalize_until_persist; +use shared::{ + model::{ + ConfigFavouritesDto, ConfigTargetDto, ConfigTargetOptions, PlaylistGroup, PlaylistItem, PlaylistItemHeader, + PlaylistItemType, Prepare, XtreamCluster, + }, + utils::Internable, +}; +use tuliprox_core::model::ConfigTarget; + +fn live(input: &str, stream_id: &str, title: &str) -> PlaylistItem { + let mut header = PlaylistItemHeader { + input_name: input.intern(), + id: stream_id.intern(), + title: title.intern(), + name: title.intern(), + url: format!("http://{input}.invalid/live/u/p/{stream_id}.ts").intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }; + header.freeze_input_stream_id(); + header.gen_uuid(); + PlaylistItem { header } +} + +fn group(title: &str, channels: Vec) -> PlaylistGroup { + PlaylistGroup { id: 1, title: title.intern(), channels, xtream_cluster: XtreamCluster::Live } +} + +fn target_with_options(options_json: &str) -> ConfigTarget { + let mut options: ConfigTargetOptions = serde_json::from_str(options_json).expect("valid options"); + options.prepare().expect("prepared"); + let dto = ConfigTargetDto { name: "T".to_string(), options: Some(options), ..Default::default() }; + ConfigTarget::from(&dto) +} + +fn target_with_options_and_favourites(options_json: &str, group_name: &str, filter_str: &str) -> ConfigTarget { + let mut options: ConfigTargetOptions = serde_json::from_str(options_json).expect("valid options"); + options.prepare().expect("prepared"); + let mut fav = ConfigFavouritesDto { + cluster: XtreamCluster::Live, + group: group_name.intern(), + filter: filter_str.to_string(), + match_as_ascii: false, + t_filter: None, + }; + fav.prepare(None).expect("prepared filter"); + let dto = ConfigTargetDto { + name: "T".to_string(), + options: Some(options), + favourites: Some(vec![fav]), + ..Default::default() + }; + ConfigTarget::from(&dto) +} + +#[tokio::test] +async fn favourites_copy_the_group_item_instead_of_joining_the_group() { + let target = target_with_options_and_favourites( + r#"{"channel_groups":{"enabled":true}}"#, + "My Favourites", + r#"Caption ~ "ZDF.*""#, + ); + let playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])]; + let (finalized, pending) = run_finalize_until_persist(&target, playlist).await; + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].members.len(), 2); + let fav = finalized.iter().find(|g| g.title.as_ref() == "My Favourites").expect("favourites group"); + assert_eq!(fav.channels.len(), 1); + assert_eq!(fav.channels[0].header.channel_group_uuid, Some(pending[0].group_uuid)); +} + +#[tokio::test] +async fn grouping_runs_before_deduplicate_and_dedup_ignores_group_members() { + let target = target_with_options(r#"{"channel_groups":{"enabled":true},"deduplicate":{}}"#); + let playlist = vec![group( + "News", + vec![ + live("solis", "1", "ZDF HD"), + live("korath", "7", "ZDF FHD"), + live("veyra", "3", "ARD HD"), + live("veyra", "4", "ARD"), + ], + )]; + let (finalized, pending) = run_finalize_until_persist(&target, playlist).await; + assert_eq!(pending.len(), 2); + assert_eq!(finalized[0].channels.len(), 2); +} diff --git a/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs b/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs index 8668e1a77..d86727c1b 100644 --- a/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs +++ b/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs @@ -15,6 +15,7 @@ use tuliprox_core::{ model::{ApiProxyConfig, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, SourcesConfig}, utils::FileLockManager, }; +use tuliprox_repository::persist_playlist_views; pub(super) fn app_config(storage_dir: &Path) -> Arc { Arc::new(AppConfig { diff --git a/backend/processing/src/processor/playlist/tests/curation_selection.rs b/backend/processing/src/processor/playlist/tests/curation_selection.rs index fb4c5cedc..5b5c26c92 100644 --- a/backend/processing/src/processor/playlist/tests/curation_selection.rs +++ b/backend/processing/src/processor/playlist/tests/curation_selection.rs @@ -69,7 +69,7 @@ fn curation_eligible_catalog_is_merged_and_deduplicated_before_matching() { ]; let mut step = StepMeasure::new("test", |_, _| {}); - let eligible = prepare_eligible_catalog(&target, playlist, &mut step); + let eligible = prepare_eligible_catalog(&target, playlist, &std::collections::HashSet::new(), &mut step); assert_eq!(eligible.len(), 1); assert_eq!(eligible[0].channels.len(), 1); diff --git a/backend/processing/src/processor/playlist/tests/identity_serialization.rs b/backend/processing/src/processor/playlist/tests/identity_serialization.rs index 6d8374a1c..e92e3fe2c 100644 --- a/backend/processing/src/processor/playlist/tests/identity_serialization.rs +++ b/backend/processing/src/processor/playlist/tests/identity_serialization.rs @@ -220,7 +220,7 @@ fn legacy_messagepack_playlist_items_default_missing_input_stream_id() { }, }; - let header_bytes = serialize_without_trailing_fields(&source.header, &[0xc0, 0xa0]); + let header_bytes = serialize_without_trailing_fields(&source.header, &[0xc0, 0xc0, 0xc0, 0xa0]); let decoded_header: PlaylistItemHeader = rmp_serde::from_slice(&header_bytes).expect("legacy header should deserialize"); assert!(decoded_header.input_stream_id.is_empty()); @@ -259,8 +259,9 @@ fn previous_messagepack_playlist_items_default_missing_upstream_user_agent() { }, }; - let header: PlaylistItemHeader = rmp_serde::from_slice(&serialize_without_trailing_fields(&source.header, &[0xc0])) - .expect("previous header should deserialize"); + let header: PlaylistItemHeader = + rmp_serde::from_slice(&serialize_without_trailing_fields(&source.header, &[0xc0, 0xc0, 0xc0])) + .expect("previous header should deserialize"); let m3u: M3uPlaylistItem = rmp_serde::from_slice(&serialize_without_trailing_fields(&M3uPlaylistItem::from(&source), &[0xc0])) .expect("previous M3U item should deserialize"); diff --git a/backend/processing/src/processor/stream_probe.rs b/backend/processing/src/processor/stream_probe.rs index ab401b97c..f75fca79c 100644 --- a/backend/processing/src/processor/stream_probe.rs +++ b/backend/processing/src/processor/stream_probe.rs @@ -289,11 +289,21 @@ async fn prepare_generic_stream_metadata( } let (raw_video, raw_audio, stats) = match probe_data { - ProbeUrlOutcome::Success(_quality, raw_video, raw_audio, stats) => ( - raw_video.map(|value| Arc::::from(value.to_string())), - raw_audio.map(|value| Arc::::from(value.to_string())), - stats, - ), + ProbeUrlOutcome::Success(_quality, raw_video, raw_audio, stats) => { + let rv = raw_video.map(|value| Arc::::from(value.to_string())); + let ra = raw_audio.map(|value| Arc::::from(value.to_string())); + if item_type.is_live() { + let bitrate_kbps = stats.bitrate.map_or(0, |b| b / 1000); + events.observe_stream_probe( + Arc::clone(&input.name), + Arc::from(unique_id), + rv.as_deref(), + ra.as_deref(), + bitrate_kbps, + ); + } + (rv, ra, stats) + } // The three failure arms are the only place the *reason* survives: // both outcome enums above collapse to a single `ProbeFailed`, so a // caller cannot tell a dead provider from a cancelled shutdown. The diff --git a/backend/repository/src/channel_group_repository.rs b/backend/repository/src/channel_group_repository.rs new file mode 100644 index 000000000..6dac57c1b --- /dev/null +++ b/backend/repository/src/channel_group_repository.rs @@ -0,0 +1,616 @@ +use crate::{ + bplustree::{BPlusTree, BPlusTreeError, BPlusTreeQuery}, + storage_const::{FILE_CHANNEL_GROUPS, FILE_CHANNEL_GROUPS_ALIAS, FILE_CHANNEL_GROUPS_MEMBER}, + target_bouquet_repository::sanitize_target_prefix, +}; +use log::debug; +use shared::{ + error::TuliproxError, + model::{ + ChannelGroupOverridesDto, ChannelGroupPolicy, ChannelMemberRefDto, PlaylistItem, PlaylistItemType, + StreamProperties, UUIDType, VirtualId, XtreamCluster, CHANNEL_GROUP_OVERRIDES_VERSION, + }, +}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; +use tuliprox_core::{ + model::{AppConfig, ConfigTarget}, + utils::{remove_file_if_exists, write_text_file_atomic}, +}; + +pub const CHANNEL_GROUP_MUTATION_LOCK: &str = "channel_group:mutation"; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum ChannelGroupOrigin { + Auto, + Manual, + AutoEdited, +} + +#[derive(Debug, Clone)] +pub struct PendingChannelGroup { + pub group_uuid: UUIDType, + pub key: Arc, + pub origin: ChannelGroupOrigin, + pub policy: ChannelGroupPolicy, + pub members: Vec, + pub orphaned: Vec, + pub epg_split: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct MemberArchive { + pub duration_days: u32, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct GroupMember { + pub member_vid: VirtualId, + pub input_name: Arc, + pub provider_id: u32, + pub input_stream_id: Arc, + pub url: Arc, + pub item_type: PlaylistItemType, + pub cluster: XtreamCluster, + pub title: Arc, + pub archive: Option, + pub epg_channel_id: Option>, + pub additional_properties: Option, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct ChannelGroupRecord { + pub group_vid: VirtualId, + pub key: Arc, + pub origin: ChannelGroupOrigin, + pub policy: ChannelGroupPolicy, + pub members: Vec, + pub orphaned: Vec, +} + +pub struct ChannelGroupStagedPublication { + target_path: PathBuf, + staged_groups: Option, + staged_members: Option, + staged_aliases: Option, +} + +impl ChannelGroupStagedPublication { + pub fn discard(self) { drop(self); } + + pub fn publish(mut self) -> std::io::Result<()> { + if let Some(p) = self.staged_groups.take() { + std::fs::rename(&p, self.target_path.join(FILE_CHANNEL_GROUPS))?; + } else { + let _ = remove_file_if_exists(&self.target_path.join(FILE_CHANNEL_GROUPS)); + } + + if let Some(p) = self.staged_members.take() { + std::fs::rename(&p, self.target_path.join(FILE_CHANNEL_GROUPS_MEMBER))?; + } else { + let _ = remove_file_if_exists(&self.target_path.join(FILE_CHANNEL_GROUPS_MEMBER)); + } + + if let Some(p) = self.staged_aliases.take() { + std::fs::rename(&p, self.target_path.join(FILE_CHANNEL_GROUPS_ALIAS))?; + } else { + let _ = remove_file_if_exists(&self.target_path.join(FILE_CHANNEL_GROUPS_ALIAS)); + } + + Ok(()) + } +} + +impl Drop for ChannelGroupStagedPublication { + fn drop(&mut self) { + if let Some(p) = &self.staged_groups { + let _ = remove_file_if_exists(p); + } + if let Some(p) = &self.staged_members { + let _ = remove_file_if_exists(p); + } + if let Some(p) = &self.staged_aliases { + let _ = remove_file_if_exists(p); + } + } +} + +pub struct ChannelGroupStore; + +impl ChannelGroupStore { + pub fn write_staged( + target_path: &Path, + records: &[ChannelGroupRecord], + aliases: &[(VirtualId, VirtualId)], + ) -> std::io::Result { + if records.is_empty() { + return Ok(ChannelGroupStagedPublication { + target_path: target_path.to_path_buf(), + staged_groups: None, + staged_members: None, + staged_aliases: None, + }); + } + + let stage_suffix = format!(".stage-{}", uuid::Uuid::new_v4()); + let staged_groups_path = target_path.join(format!("{FILE_CHANNEL_GROUPS}{stage_suffix}")); + let mut groups_tree = BPlusTree::::new(); + for record in records { + groups_tree.insert(record.group_vid, record.clone()); + } + if let Err(err) = groups_tree.store(&staged_groups_path) { + let _ = remove_file_if_exists(&staged_groups_path); + return Err(err); + } + + let staged_members_path = target_path.join(format!("{FILE_CHANNEL_GROUPS_MEMBER}{stage_suffix}")); + let mut members_tree = BPlusTree::::new(); + for record in records { + for member in &record.members { + members_tree.insert(member.member_vid, record.group_vid); + } + } + if let Err(err) = members_tree.store(&staged_members_path) { + let _ = remove_file_if_exists(&staged_groups_path); + let _ = remove_file_if_exists(&staged_members_path); + return Err(err); + } + + let staged_aliases_path = if aliases.is_empty() { + None + } else { + let path = target_path.join(format!("{FILE_CHANNEL_GROUPS_ALIAS}{stage_suffix}")); + let mut aliases_tree = BPlusTree::::new(); + for (alias_vid, group_vid) in aliases { + aliases_tree.insert(*alias_vid, *group_vid); + } + if let Err(err) = aliases_tree.store(&path) { + let _ = remove_file_if_exists(&staged_groups_path); + let _ = remove_file_if_exists(&staged_members_path); + let _ = remove_file_if_exists(&path); + return Err(err); + } + Some(path) + }; + + Ok(ChannelGroupStagedPublication { + target_path: target_path.to_path_buf(), + staged_groups: Some(staged_groups_path), + staged_members: Some(staged_members_path), + staged_aliases: staged_aliases_path, + }) + } + + pub fn write( + target_path: &Path, + records: &[ChannelGroupRecord], + aliases: &[(VirtualId, VirtualId)], + ) -> std::io::Result<()> { + Self::write_staged(target_path, records, aliases)?.publish() + } + + pub fn delete(target_path: &Path) -> std::io::Result<()> { + let _ = remove_file_if_exists(&target_path.join(FILE_CHANNEL_GROUPS)); + let _ = remove_file_if_exists(&target_path.join(FILE_CHANNEL_GROUPS_MEMBER)); + let _ = remove_file_if_exists(&target_path.join(FILE_CHANNEL_GROUPS_ALIAS)); + Ok(()) + } + + pub fn read_alias(target_path: &Path, vid: VirtualId) -> std::io::Result> { + let db_path = target_path.join(FILE_CHANNEL_GROUPS_ALIAS); + if !db_path.exists() { + return Ok(None); + } + let mut query = match BPlusTreeQuery::::try_new(&db_path) { + Ok(q) => q, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err), + }; + query.query(&vid).map_err(BPlusTreeError::to_io) + } + + pub fn read_group(target_path: &Path, group_vid: VirtualId) -> std::io::Result> { + let db_path = target_path.join(FILE_CHANNEL_GROUPS); + if !db_path.exists() { + return Ok(None); + } + let mut query = match BPlusTreeQuery::::try_new(&db_path) { + Ok(q) => q, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err), + }; + query.query(&group_vid).map_err(BPlusTreeError::to_io) + } + + pub fn read_member( + target_path: &Path, + member_vid: VirtualId, + ) -> std::io::Result> { + let db_path = target_path.join(FILE_CHANNEL_GROUPS_MEMBER); + if !db_path.exists() { + return Ok(None); + } + let mut member_query = match BPlusTreeQuery::::try_new(&db_path) { + Ok(q) => q, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err), + }; + let Some(group_vid) = member_query.query(&member_vid).map_err(BPlusTreeError::to_io)? else { + return Ok(None); + }; + drop(member_query); + let Some(record) = Self::read_group(target_path, group_vid)? else { + return Ok(None); + }; + let Some(idx) = record.members.iter().position(|m| m.member_vid == member_vid) else { + return Ok(None); + }; + Ok(Some((record, idx))) + } + + pub fn read_all_groups(target_path: &Path) -> std::io::Result> { + let db_path = target_path.join(FILE_CHANNEL_GROUPS); + if !db_path.exists() { + return Ok(Vec::new()); + } + let mut query = match BPlusTreeQuery::::try_new(&db_path) { + Ok(q) => q, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(err), + }; + let mut records = Vec::new(); + for entry in query.iter() { + let (_key, record) = entry?; + records.push(record); + } + Ok(records) + } + + pub fn read_all_aliases(target_path: &Path) -> std::io::Result> { + let db_path = target_path.join(FILE_CHANNEL_GROUPS_ALIAS); + if !db_path.exists() { + return Ok(HashMap::new()); + } + let mut query = match BPlusTreeQuery::::try_new(&db_path) { + Ok(q) => q, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(HashMap::new()), + Err(err) => return Err(err), + }; + let mut aliases = HashMap::new(); + for entry in query.iter() { + let (alias_vid, group_vid) = entry?; + aliases.insert(alias_vid, group_vid); + } + Ok(aliases) + } + + pub fn load_target_data(target_path: &Path, generation: u64) -> std::io::Result { + let records = Self::read_all_groups(target_path)?; + let aliases = Self::read_all_aliases(target_path)?; + let mut groups = HashMap::with_capacity(records.len()); + let mut members = HashMap::new(); + for record in records { + let group_vid = record.group_vid; + for (idx, member) in record.members.iter().enumerate() { + members.insert(member.member_vid, (group_vid, idx)); + } + groups.insert(group_vid, Arc::new(record)); + } + Ok(ChannelGroupTargetData { generation, groups, members, aliases }) + } +} + +#[derive(Debug, Clone, Default)] +pub struct ChannelGroupTargetData { + pub generation: u64, + pub groups: HashMap>, + pub members: HashMap, + pub aliases: HashMap, +} + +pub struct ChannelGroupCache { + targets: tokio::sync::RwLock>>, +} + +impl Default for ChannelGroupCache { + fn default() -> Self { Self::new() } +} + +impl ChannelGroupCache { + #[must_use] + pub fn new() -> Self { Self { targets: tokio::sync::RwLock::new(HashMap::new()) } } + + pub async fn get_or_load( + &self, + app_config: &Arc, + playlists: &crate::PlaylistStorageState, + target: &ConfigTarget, + ) -> Arc { + let current_generation = playlists.channel_group_generation(); + { + let guard = self.targets.read().await; + if let Some(cached) = guard.get(&target.name) { + if cached.generation == current_generation { + return Arc::clone(cached); + } + } + } + + let config = app_config.config.load(); + let Some(target_path) = crate::get_target_storage_path(&config, &target.name) else { + return Arc::new(ChannelGroupTargetData { + generation: current_generation, + groups: HashMap::new(), + members: HashMap::new(), + aliases: HashMap::new(), + }); + }; + + let file_lock = app_config.file_locks.read_lock(&target_path).await; + let loaded = tokio::task::spawn_blocking(move || { + let _guard = file_lock; + ChannelGroupStore::load_target_data(&target_path, current_generation) + }) + .await + .unwrap_or_else(|_| { + Ok(ChannelGroupTargetData { + generation: current_generation, + groups: HashMap::new(), + members: HashMap::new(), + aliases: HashMap::new(), + }) + }) + .unwrap_or_else(|_| ChannelGroupTargetData { + generation: current_generation, + groups: HashMap::new(), + members: HashMap::new(), + aliases: HashMap::new(), + }); + + let arc_data = Arc::new(loaded); + let mut guard = self.targets.write().await; + guard.insert(target.name.clone(), Arc::clone(&arc_data)); + arc_data + } +} + +#[derive(Debug)] +pub enum ChannelGroupSaveError { + RevisionMismatch { current: u64 }, + Io(TuliproxError), +} + +impl std::fmt::Display for ChannelGroupSaveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::RevisionMismatch { current } => write!(f, "revision mismatch: current is {current}"), + Self::Io(err) => write!(f, "{err}"), + } + } +} + +impl std::error::Error for ChannelGroupSaveError {} + +impl From for ChannelGroupSaveError { + fn from(err: TuliproxError) -> Self { Self::Io(err) } +} + +/// Returns the channel group storage directory derived from `config_path`. +pub fn channel_group_overrides_dir(config_path: &Path) -> PathBuf { config_path.join("channel_groups") } + +/// Derives the canonical filename for a target's channel group overrides. +pub fn channel_group_overrides_file_name(target_name: &str) -> String { + let prefix = sanitize_target_prefix(target_name); + let hash = blake3::hash(target_name.as_bytes()); + let hex = hash.to_hex(); + format!("{}--{}.yml", prefix, &hex[..12]) +} + +/// Derives the full path to a target's channel group overrides file. +pub fn channel_group_overrides_path(config_path: &Path, target_name: &str) -> PathBuf { + channel_group_overrides_dir(config_path).join(channel_group_overrides_file_name(target_name)) +} + +/// Loads a target's channel group overrides from disk. Returns `Ok(None)` if the file does not exist. +/// Fails closed with an error on corruption, target mismatch, or unsupported version. +pub async fn load_channel_group_overrides( + app_config: &AppConfig, + target_name: &str, +) -> Result, TuliproxError> { + let paths = app_config.paths.load(); + let config_path = Path::new(&paths.config_path); + let file_path = channel_group_overrides_path(config_path, target_name); + + let _lock = app_config.file_locks.read_lock(&file_path).await; + + let content = match tokio::fs::read_to_string(&file_path).await { + Ok(c) => c, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => { + return Err(TuliproxError::ChannelGroup(format!( + "Failed to read channel group overrides file {}: {err}", + file_path.display() + ))); + } + }; + + let mut dto: ChannelGroupOverridesDto = serde_saphyr::from_str(&content).map_err(|err| { + TuliproxError::ChannelGroup(format!( + "Failed to parse channel group overrides YAML in {}: {err}", + file_path.display() + )) + })?; + + if dto.version != CHANNEL_GROUP_OVERRIDES_VERSION { + return Err(TuliproxError::ChannelGroup(format!( + "Unsupported channel group overrides version {} in {}", + dto.version, + file_path.display() + ))); + } + + if dto.target != target_name { + return Err(TuliproxError::ChannelGroup(format!( + "Target name mismatch in {}: file contains target '{}', expected '{target_name}'", + file_path.display(), + dto.target + ))); + } + + dto.canonicalize(); + Ok(Some(dto)) +} + +/// Saves channel group overrides with optimistic concurrency control. +/// Returns the new revision on success. +pub async fn save_channel_group_overrides( + app_config: &AppConfig, + target_name: &str, + mut dto: ChannelGroupOverridesDto, + expected_revision: u64, +) -> Result { + let paths = app_config.paths.load(); + let config_path = Path::new(&paths.config_path); + let file_path = channel_group_overrides_path(config_path, target_name); + + let _mutation_lock = app_config.file_locks.write_lock_str(CHANNEL_GROUP_MUTATION_LOCK).await; + let _file_lock = app_config.file_locks.write_lock(&file_path).await; + + let current = match tokio::fs::read_to_string(&file_path).await { + Ok(content) => { + let existing: ChannelGroupOverridesDto = serde_saphyr::from_str(&content).map_err(|err| { + TuliproxError::ChannelGroup(format!( + "Failed to parse channel group overrides YAML in {}: {err}", + file_path.display() + )) + })?; + if existing.target != target_name { + return Err(ChannelGroupSaveError::Io(TuliproxError::ChannelGroup(format!( + "Target mismatch in existing overrides file {}: '{}' != '{target_name}'", + file_path.display(), + existing.target + )))); + } + existing.revision + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => 0, + Err(err) => { + return Err(ChannelGroupSaveError::Io(TuliproxError::ChannelGroup(format!( + "Failed to read channel group overrides {}: {err}", + file_path.display() + )))); + } + }; + + if current != expected_revision { + return Err(ChannelGroupSaveError::RevisionMismatch { current }); + } + + let next_revision = current + 1; + dto.version = CHANNEL_GROUP_OVERRIDES_VERSION; + dto.target = target_name.to_string(); + dto.revision = next_revision; + dto.canonicalize(); + + let yaml = serde_saphyr::to_string(&dto).map_err(|err| { + TuliproxError::ChannelGroup(format!("Failed to serialize channel group overrides to YAML: {err}")) + })?; + + write_text_file_atomic(&file_path, &yaml).await.map_err(|err| { + TuliproxError::ChannelGroup(format!("Failed to write channel group overrides {}: {err}", file_path.display())) + })?; + + debug!( + "Saved channel group overrides for '{}' (revision {}) to {}", + target_name, + next_revision, + file_path.display() + ); + + Ok(next_revision) +} + +#[cfg(test)] +mod tests { + use super::*; + use arc_swap::{ArcSwap, ArcSwapOption}; + use shared::model::ConfigPaths; + use std::sync::Arc; + use tuliprox_core::{ + model::{Config, MediaToolCapabilities, SourcesConfig}, + utils::FileLockManager, + }; + + fn test_app_config(config_path: &str) -> AppConfig { + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: config_path.to_string(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } + } + + fn test_app_config_with_config_dir() -> (AppConfig, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let cfg = test_app_config(dir.path().to_str().expect("utf-8 path")); + (cfg, dir) + } + + #[tokio::test] + async fn save_then_load_round_trips_and_bumps_revision() { + let (app_config, _dir) = test_app_config_with_config_dir(); + let dto = ChannelGroupOverridesDto::new("Main TV"); + let revision = save_channel_group_overrides(&app_config, "Main TV", dto, 0).await.expect("saves"); + assert_eq!(revision, 1); + let loaded = load_channel_group_overrides(&app_config, "Main TV").await.expect("loads").expect("exists"); + assert_eq!(loaded.revision, 1); + assert_eq!(loaded.target, "Main TV"); + } + + #[tokio::test] + async fn stale_revision_is_rejected() { + let (app_config, _dir) = test_app_config_with_config_dir(); + save_channel_group_overrides(&app_config, "T", ChannelGroupOverridesDto::new("T"), 0).await.expect("saves"); + let err = + save_channel_group_overrides(&app_config, "T", ChannelGroupOverridesDto::new("T"), 0).await.unwrap_err(); + assert!(matches!(err, ChannelGroupSaveError::RevisionMismatch { current: 1 })); + } + + #[tokio::test] + async fn target_mismatch_and_unknown_version_fail_closed() { + let (app_config, _dir) = test_app_config_with_config_dir(); + let path = channel_group_overrides_path(Path::new(&app_config.paths.load().config_path), "T"); + tokio::fs::create_dir_all(path.parent().unwrap()).await.unwrap(); + tokio::fs::write(&path, "version: 1\ntarget: Other\nrevision: 3\n").await.unwrap(); + assert!(load_channel_group_overrides(&app_config, "T").await.is_err()); + tokio::fs::write(&path, "version: 9\ntarget: T\nrevision: 3\n").await.unwrap(); + assert!(load_channel_group_overrides(&app_config, "T").await.is_err()); + } + + #[tokio::test] + async fn missing_file_is_none() { + let (app_config, _dir) = test_app_config_with_config_dir(); + assert!(load_channel_group_overrides(&app_config, "T").await.expect("ok").is_none()); + } +} diff --git a/backend/repository/src/lib.rs b/backend/repository/src/lib.rs index 9a330be1e..9a5f493c5 100644 --- a/backend/repository/src/lib.rs +++ b/backend/repository/src/lib.rs @@ -6,6 +6,7 @@ mod target_id_mapping; // `utils`, which must not depend on this layer. mod alias_repository; mod api_user_recovery; +pub mod channel_group_repository; mod db_viewer; mod epg_group_index; mod epg_repository; @@ -42,6 +43,7 @@ mod xtream_playlist_iterator; mod xtream_repository; pub use alias_repository::*; +pub use channel_group_repository::*; pub use db_viewer::*; pub use epg_group_index::*; pub use epg_repository::*; @@ -74,6 +76,8 @@ pub use stream_history::*; pub use strm_repository::*; pub use target_bouquet_repository::*; pub use target_id_mapping::*; +pub mod target_stream_item; +pub use target_stream_item::*; // The B+Tree storage engine is its own package. Aliased under its historical // module name so every `crate::bplustree::X` path keeps resolving. pub use tuliprox_btree as bplustree; diff --git a/backend/repository/src/playlist_mem_cache.rs b/backend/repository/src/playlist_mem_cache.rs index d0d64644e..b2fa85b7b 100644 --- a/backend/repository/src/playlist_mem_cache.rs +++ b/backend/repository/src/playlist_mem_cache.rs @@ -1,6 +1,9 @@ use super::{BPlusTree, VirtualIdRecord}; use shared::model::{M3uPlaylistItem, PlaylistItem, VirtualId, XtreamCluster, XtreamPlaylistItem}; -use std::collections::HashMap; +use std::{ + collections::HashMap, + sync::atomic::{AtomicU64, Ordering}, +}; use tokio::sync::RwLock; use tuliprox_core::model::ConfigTarget; @@ -27,6 +30,8 @@ pub type TargetPlaylistStorageMap = HashMap; pub struct PlaylistStorageState { pub data: RwLock, + pub channel_group_generation: AtomicU64, + pub channel_groups: crate::ChannelGroupCache, } impl Default for PlaylistStorageState { @@ -34,7 +39,20 @@ impl Default for PlaylistStorageState { } impl PlaylistStorageState { - pub fn new() -> Self { Self { data: RwLock::new(HashMap::new()) } } + pub fn new() -> Self { + Self { + data: RwLock::new(HashMap::new()), + channel_group_generation: AtomicU64::new(0), + channel_groups: crate::ChannelGroupCache::new(), + } + } + + pub fn bump_channel_group_generation(&self) -> u64 { + self.channel_group_generation.fetch_add(1, Ordering::SeqCst) + 1 + } + + #[must_use] + pub fn channel_group_generation(&self) -> u64 { self.channel_group_generation.load(Ordering::SeqCst) } /// Replaces one target only after its complete persisted state has been loaded. pub(crate) async fn replace_target(&self, target_name: &str, storage: TargetPlaylistStorage) { diff --git a/backend/repository/src/playlist_repository.rs b/backend/repository/src/playlist_repository.rs index f79cc62e4..f19b1b4d4 100644 --- a/backend/repository/src/playlist_repository.rs +++ b/backend/repository/src/playlist_repository.rs @@ -37,6 +37,7 @@ use persist::persist_playlist_with_mode; #[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] pub use persist::{ load_input_media_server_playlist, persist_input_media_server_playlist, persist_playlist, persist_playlist_views, + persist_playlist_views_with_groups, }; #[cfg(test)] use publication::TargetCacheReloadStage; diff --git a/backend/repository/src/playlist_repository/episode.rs b/backend/repository/src/playlist_repository/episode.rs index 3fd5338d0..a5e7bf8bb 100644 --- a/backend/repository/src/playlist_repository/episode.rs +++ b/backend/repository/src/playlist_repository/episode.rs @@ -37,41 +37,47 @@ pub(super) fn normalize_target_playlist_epg_ids( } } +pub(super) fn assign_item_identity( + header: &mut PlaylistItemHeader, + source_ordinal: &mut u32, + target_id_mapping: &mut TargetIdMapping, +) { + *source_ordinal += 1; + header.source_ordinal = *source_ordinal; + let provider_id = header.get_provider_id().unwrap_or_default(); + if provider_id == 0 { + header.item_type = match (is_hls_url(&header.url), header.item_type) { + (true, _) => PlaylistItemType::LiveHls, + (false, PlaylistItemType::Live) => { + if is_dash_url(&header.url) { + PlaylistItemType::LiveDash + } else { + PlaylistItemType::LiveUnknown + } + } + _ => header.item_type, + }; + } + + let uuid = header.get_uuid(); + let item_type = header.item_type; + let parent_virtual_id = if item_type.is_series() { + target_id_mapping.get_parent_virtual_id_by_uuid(uuid).unwrap_or_default() + } else { + VirtualId::default() + }; + header.virtual_id = target_id_mapping.get_and_update_virtual_id(uuid, provider_id, item_type, parent_virtual_id); +} + pub(super) fn prepare_target_playlist_for_persistence( playlist: &mut [PlaylistGroup], target: &ConfigTarget, target_id_mapping: &mut TargetIdMapping, + source_ordinal: &mut u32, ) { - let mut source_ordinal: u32 = 0; for group in playlist.iter_mut() { for channel in &mut group.channels { - let header = &mut channel.header; - source_ordinal += 1; - header.source_ordinal = source_ordinal; - let provider_id = header.get_provider_id().unwrap_or_default(); - if provider_id == 0 { - header.item_type = match (is_hls_url(&header.url), header.item_type) { - (true, _) => PlaylistItemType::LiveHls, - (false, PlaylistItemType::Live) => { - if is_dash_url(&header.url) { - PlaylistItemType::LiveDash - } else { - PlaylistItemType::LiveUnknown - } - } - _ => header.item_type, - }; - } - - let uuid = header.get_uuid(); - let item_type = header.item_type; - let parent_virtual_id = if item_type.is_series() { - target_id_mapping.get_parent_virtual_id_by_uuid(uuid).unwrap_or_default() - } else { - VirtualId::default() - }; - header.virtual_id = - target_id_mapping.get_and_update_virtual_id(uuid, provider_id, item_type, parent_virtual_id); + assign_item_identity(&mut channel.header, source_ordinal, target_id_mapping); } } diff --git a/backend/repository/src/playlist_repository/persist.rs b/backend/repository/src/playlist_repository/persist.rs index 4c4ac8543..7cce42035 100644 --- a/backend/repository/src/playlist_repository/persist.rs +++ b/backend/repository/src/playlist_repository/persist.rs @@ -1,5 +1,5 @@ use super::{ - episode::prepare_target_playlist_for_persistence, + episode::{assign_item_identity, prepare_target_playlist_for_persistence}, get_target_id_mapping, publication::{ persist_xtream_target_playlist, playlist_has_items, prepare_target_output_playlists, @@ -10,9 +10,13 @@ use super::{ }; use crate::{ ensure_target_storage_path, epg_write_for_target, load_input_local_library_playlist, m3u_write_playlist, - persist_input_library_playlist, write_strm_playlist, + persist_input_library_playlist, write_strm_playlist, ChannelGroupRecord, ChannelGroupStore, GroupMember, + MemberArchive, PendingChannelGroup, TargetIdMapping, +}; +use shared::{ + error::TuliproxError, + model::{PlaylistGroup, StreamProperties, VirtualId}, }; -use shared::{error::TuliproxError, model::PlaylistGroup}; use std::{path::Path, sync::Arc}; use tuliprox_core::model::{AppConfig, ConfigTarget, Epg, TargetOutput}; @@ -44,6 +48,30 @@ pub async fn persist_playlist_views( target: &ConfigTarget, playlist_state: Option<&Arc>, options: TargetPlaylistPersistOptions, +) -> Result<(), Vec> { + persist_playlist_views_with_groups( + app_config, + base_playlist, + xtream_playlist, + epg, + target, + playlist_state, + options, + &mut [], + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn persist_playlist_views_with_groups( + app_config: &Arc, + base_playlist: &mut [PlaylistGroup], + xtream_playlist: Option<&mut [PlaylistGroup]>, + epg: Option<&Epg>, + target: &ConfigTarget, + playlist_state: Option<&Arc>, + options: TargetPlaylistPersistOptions, + channel_groups: &mut [PendingChannelGroup], ) -> Result<(), Vec> { persist_playlist_views_with_mode( app_config, @@ -54,6 +82,7 @@ pub async fn persist_playlist_views( playlist_state, options, TargetPersistenceMode::Persist, + channel_groups, ) .await } @@ -67,8 +96,83 @@ pub(super) async fn persist_playlist_with_mode( options: TargetPlaylistPersistOptions, persistence_mode: TargetPersistenceMode, ) -> Result<(), Vec> { - persist_playlist_views_with_mode(app_config, playlist, None, epg, target, playlist_state, options, persistence_mode) - .await + persist_playlist_views_with_mode( + app_config, + playlist, + None, + epg, + target, + playlist_state, + options, + persistence_mode, + &mut [], + ) + .await +} + +fn collect_channel_group_aliases( + playlist: &[PlaylistGroup], + mapping: &TargetIdMapping, + aliases: &mut Vec<(VirtualId, VirtualId)>, +) { + for group in playlist { + for channel in &group.channels { + if let Some(cg_uuid) = channel.header.channel_group_uuid { + if channel.header.uuid != cg_uuid { + if let Some(group_vid) = mapping.get_virtual_id_by_uuid(&cg_uuid) { + aliases.push((channel.header.virtual_id, group_vid)); + } + } + } + } + } +} + +fn build_channel_group_records( + channel_groups: &[PendingChannelGroup], + mapping: &TargetIdMapping, +) -> Vec { + channel_groups + .iter() + .map(|pending| { + let group_vid = mapping.get_virtual_id_by_uuid(&pending.group_uuid).unwrap_or_default(); + let members = pending + .members + .iter() + .map(|member| { + let archive = if let Some(StreamProperties::Live(live)) = &member.header.additional_properties { + live.tv_archive_duration + .and_then(|d| u32::try_from(d).ok()) + .map(|duration_days| MemberArchive { duration_days }) + } else { + None + }; + GroupMember { + member_vid: member.header.virtual_id, + input_name: Arc::clone(&member.header.input_name), + provider_id: shared::utils::get_provider_id(&member.header.id, &member.header.url) + .unwrap_or_default(), + input_stream_id: Arc::clone(&member.header.input_stream_id), + url: Arc::clone(&member.header.url), + item_type: member.header.item_type, + cluster: member.header.xtream_cluster, + title: Arc::clone(&member.header.title), + archive, + epg_channel_id: member.header.epg_channel_id.as_ref().map(Arc::clone), + additional_properties: member.header.additional_properties.clone(), + } + }) + .collect(); + ChannelGroupRecord { + group_vid, + key: Arc::clone(&pending.key), + origin: pending.origin, + policy: pending.policy, + members, + orphaned: pending.orphaned.clone(), + } + }) + .collect() } #[allow(clippy::too_many_arguments, clippy::too_many_lines)] @@ -81,6 +185,7 @@ async fn persist_playlist_views_with_mode( playlist_state: Option<&Arc>, options: TargetPlaylistPersistOptions, persistence_mode: TargetPersistenceMode, + channel_groups: &mut [PendingChannelGroup], ) -> Result<(), Vec> { let playlist_is_empty = !playlist_has_items(base_playlist) && xtream_playlist.as_deref().is_none_or(|playlist| !playlist_has_items(playlist)); @@ -100,9 +205,16 @@ async fn persist_playlist_views_with_mode( Err(err) => return Err(vec![err]), }; - prepare_target_playlist_for_persistence(base_playlist, target, &mut target_id_mapping); + let mut source_ordinal = 0u32; + prepare_target_playlist_for_persistence(base_playlist, target, &mut target_id_mapping, &mut source_ordinal); if let Some(xtream_view) = xtream_playlist.as_deref_mut() { - prepare_target_playlist_for_persistence(xtream_view, target, &mut target_id_mapping); + prepare_target_playlist_for_persistence(xtream_view, target, &mut target_id_mapping, &mut source_ordinal); + } + + for pending in channel_groups.iter_mut() { + for member in &mut pending.members { + assign_item_identity(&mut member.header, &mut source_ordinal, &mut target_id_mapping); + } } let mut prepared_outputs = prepare_target_output_playlists(target, base_playlist, xtream_playlist.as_deref()); @@ -113,6 +225,26 @@ async fn persist_playlist_views_with_mode( return Err(vec![error]); } + let records = build_channel_group_records(channel_groups, &target_id_mapping); + let mut aliases = Vec::new(); + collect_channel_group_aliases(base_playlist, &target_id_mapping, &mut aliases); + if let Some(xtream_view) = xtream_playlist.as_deref() { + collect_channel_group_aliases(xtream_view, &target_id_mapping, &mut aliases); + } + + let staged_groups = match ChannelGroupStore::write_staged(&target_path, &records, &aliases) { + Ok(staged) => staged, + Err(err) => { + target_id_mapping.discard_unpersisted_changes(); + drop(target_id_mapping); + drop(file_lock); + return Err(vec![TuliproxError::RepositoryPlaylist(format!( + "Failed to stage channel groups for target '{}': {err}", + target.name + ))]); + } + }; + for (output, prepared_output) in target.output.iter().zip(&mut prepared_outputs) { let output_publication_plan = options.publication_plan.with_output_filter(output.filter().is_some()); let source_playlist = match output { @@ -181,6 +313,19 @@ async fn persist_playlist_views_with_mode( drop(target_id_mapping); drop(file_lock); + if errors.is_empty() { + if let Err(err) = staged_groups.publish() { + errors.push(TuliproxError::RepositoryPlaylist(format!( + "Failed to publish channel groups for target '{}': {err}", + target.name + ))); + } else if let Some(playlist_storage) = playlist_state { + playlist_storage.bump_channel_group_generation(); + } + } else { + staged_groups.discard(); + } + if errors.is_empty() && target.use_memory_cache { match playlist_state { Some(playlist_storage) => { diff --git a/backend/repository/src/playlist_repository/tests.rs b/backend/repository/src/playlist_repository/tests.rs index 15c30194b..232039e4f 100644 --- a/backend/repository/src/playlist_repository/tests.rs +++ b/backend/repository/src/playlist_repository/tests.rs @@ -388,6 +388,7 @@ fn make_media_server_episode( } } +mod channel_groups; mod lifecycle; mod playlist; mod policy; diff --git a/backend/repository/src/playlist_repository/tests/channel_groups.rs b/backend/repository/src/playlist_repository/tests/channel_groups.rs new file mode 100644 index 000000000..fe8b4dca9 --- /dev/null +++ b/backend/repository/src/playlist_repository/tests/channel_groups.rs @@ -0,0 +1,341 @@ +use super::*; +use crate::{persist_playlist_views_with_groups, ChannelGroupOrigin, ChannelGroupStore, PendingChannelGroup}; +use shared::model::ChannelGroupPolicy; +use std::path::PathBuf; + +fn persist_test_env(_config_json: &str) -> (Arc, ConfigTarget, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let app_config = test_app_config(dir.path()); + let target = ConfigTarget::from(&ConfigTargetDto { + name: "target-groups-test".to_string(), + output: vec![TargetOutputDto::M3u(M3uTargetOutputDto { + filename: Some("playlist.m3u".to_string()), + ..M3uTargetOutputDto::default() + })], + ..ConfigTargetDto::default() + }); + (app_config, target, dir) +} + +fn persist_test_env_rejecting_empty() -> (Arc, ConfigTarget, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let app_config = test_app_config(dir.path()); + let target = ConfigTarget::from(&ConfigTargetDto { + name: "target-reject-test".to_string(), + output: vec![TargetOutputDto::M3u(M3uTargetOutputDto { + filename: Some("playlist.m3u".to_string()), + ..M3uTargetOutputDto::default() + })], + ..ConfigTargetDto::default() + }); + (app_config, target, dir) +} + +fn target_path(app_config: &AppConfig, target: &ConfigTarget) -> PathBuf { + let config = app_config.config.load(); + get_target_storage_path(&config, &target.name).expect("target storage path") +} + +fn grouped_fixture() -> (Vec, Vec) { + let group_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000001"); + let group_channel = PlaylistItem { + header: PlaylistItemHeader { + uuid: group_uuid, + id: "bbc-one".intern(), + input_stream_id: "bbc-one".intern(), + name: "BBC One".intern(), + title: "BBC One".intern(), + group: "Entertainment".intern(), + url: "http://korath/live/101".intern(), + input_name: "korath".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + channel_group_uuid: Some(group_uuid), + ..PlaylistItemHeader::default() + }, + }; + let normal_channel = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000002"), + id: "cnn".intern(), + input_stream_id: "cnn".intern(), + name: "CNN".intern(), + title: "CNN".intern(), + group: "News".intern(), + url: "http://korath/live/cnn".intern(), + input_name: "korath".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let member1 = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000011"), + id: "101".intern(), + input_stream_id: "101".intern(), + name: "BBC One FHD".intern(), + title: "BBC One FHD".intern(), + group: "Entertainment".intern(), + url: "http://korath/live/101".intern(), + input_name: "korath".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let member2 = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000012"), + id: "201".intern(), + input_stream_id: "201".intern(), + name: "BBC One HD".intern(), + title: "BBC One HD".intern(), + group: "Entertainment".intern(), + url: "http://xandar/live/201".intern(), + input_name: "xandar".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let playlist = vec![ + PlaylistGroup { + id: 1, + title: "Entertainment".intern(), + channels: vec![group_channel], + xtream_cluster: XtreamCluster::Live, + }, + PlaylistGroup { + id: 2, + title: "News".intern(), + channels: vec![normal_channel], + xtream_cluster: XtreamCluster::Live, + }, + ]; + let pending = vec![PendingChannelGroup { + group_uuid, + key: Arc::from("bbc one"), + origin: ChannelGroupOrigin::Auto, + policy: ChannelGroupPolicy::Priority, + members: vec![member1, member2], + orphaned: vec![], + epg_split: false, + }]; + (playlist, pending) +} + +fn grouped_fixture_ungrouped() -> (Vec, Vec) { + let member1 = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000011"), + id: "101".intern(), + input_stream_id: "101".intern(), + name: "BBC One FHD".intern(), + title: "BBC One FHD".intern(), + group: "Entertainment".intern(), + url: "http://korath/live/101".intern(), + input_name: "korath".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let member2 = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000012"), + id: "201".intern(), + input_stream_id: "201".intern(), + name: "BBC One HD".intern(), + title: "BBC One HD".intern(), + group: "Entertainment".intern(), + url: "http://xandar/live/201".intern(), + input_name: "xandar".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let playlist = vec![PlaylistGroup { + id: 1, + title: "Entertainment".intern(), + channels: vec![member1, member2], + xtream_cluster: XtreamCluster::Live, + }]; + (playlist, Vec::new()) +} + +fn grouped_fixture_with_favourite_copy() -> (Vec, Vec) { + let (mut playlist, pending) = grouped_fixture(); + let group_item = &playlist[0].channels[0]; + let fav_copy = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000099"), + channel_group_uuid: Some(group_item.header.uuid), + group: "My Favourites".intern(), + ..group_item.header.clone() + }, + }; + playlist.push(PlaylistGroup { + id: 99, + title: "My Favourites".intern(), + channels: vec![fav_copy], + xtream_cluster: XtreamCluster::Live, + }); + (playlist, pending) +} + +#[tokio::test] +async fn members_get_stable_virtual_ids_and_store_is_written() { + let (app_config, target, _dir) = persist_test_env(r#"{"channel_groups":{"enabled":true}}"#); + let (mut playlist, mut pending) = grouped_fixture(); + persist_playlist_views_with_groups( + &app_config, + &mut playlist, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .expect("persists"); + let path = target_path(&app_config, &target); + let group_vid = playlist[0].channels[0].header.virtual_id; + let record = ChannelGroupStore::read_group(&path, group_vid).expect("reads").expect("exists"); + assert_eq!(record.members.len(), 2); + assert!(record.members.iter().all(|m| !m.member_vid.is_unset() && m.member_vid != group_vid)); + let (via_member, idx) = + ChannelGroupStore::read_member(&path, record.members[1].member_vid).expect("reads").expect("exists"); + assert_eq!((via_member.group_vid, idx), (group_vid, 1)); + + // second run keeps ids + let (mut playlist2, mut pending2) = grouped_fixture(); + persist_playlist_views_with_groups( + &app_config, + &mut playlist2, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut pending2, + ) + .await + .expect("persists"); + let record2 = ChannelGroupStore::read_group(&path, playlist2[0].channels[0].header.virtual_id).unwrap().unwrap(); + assert_eq!(record.members[0].member_vid, record2.members[0].member_vid); +} + +#[tokio::test] +async fn member_keeps_vid_of_previously_ungrouped_channel() { + let (app_config, target, _dir) = persist_test_env(r#"{"channel_groups":{"enabled":true}}"#); + let (mut plain, _) = grouped_fixture_ungrouped(); + persist_playlist_views_with_groups( + &app_config, + &mut plain, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut [], + ) + .await + .unwrap(); + let korath_vid_before = + plain[0].channels.iter().find(|c| c.header.input_name.as_ref() == "korath").unwrap().header.virtual_id; + let (mut playlist, mut pending) = grouped_fixture(); + persist_playlist_views_with_groups( + &app_config, + &mut playlist, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .unwrap(); + let record = + ChannelGroupStore::read_group(&target_path(&app_config, &target), playlist[0].channels[0].header.virtual_id) + .unwrap() + .unwrap(); + assert!(record.members.iter().any(|m| m.member_vid == korath_vid_before)); +} + +#[tokio::test] +async fn favourite_copy_of_group_item_is_registered_as_alias() { + let (app_config, target, _dir) = persist_test_env(r#"{"channel_groups":{"enabled":true}}"#); + let (mut playlist, mut pending) = grouped_fixture_with_favourite_copy(); + persist_playlist_views_with_groups( + &app_config, + &mut playlist, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .unwrap(); + let group_vid = playlist[0].channels[0].header.virtual_id; + let fav_vid = playlist.iter().find(|g| g.title.as_ref() == "My Favourites").unwrap().channels[0].header.virtual_id; + assert_ne!(fav_vid, group_vid); + assert_eq!(ChannelGroupStore::read_alias(&target_path(&app_config, &target), fav_vid).unwrap(), Some(group_vid)); +} + +#[tokio::test] +async fn rejected_persistence_writes_no_group_store() { + let (app_config, target, _dir) = persist_test_env_rejecting_empty(); + let mut playlist = vec![]; + let mut pending = grouped_fixture().1; + assert!(persist_playlist_views_with_groups( + &app_config, + &mut playlist, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .is_err()); + assert!(!target_path(&app_config, &target).join("channel_groups.db").exists()); +} + +#[tokio::test] +async fn target_without_groups_removes_stale_group_store() { + let (app_config, target, _dir) = persist_test_env(r#"{"channel_groups":{"enabled":true}}"#); + let (mut playlist, mut pending) = grouped_fixture(); + persist_playlist_views_with_groups( + &app_config, + &mut playlist, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .unwrap(); + let (mut plain, _) = grouped_fixture_ungrouped(); + persist_playlist_views_with_groups( + &app_config, + &mut plain, + None, + None, + &target, + None, + TargetPlaylistPersistOptions::default(), + &mut [], + ) + .await + .unwrap(); + assert!(!target_path(&app_config, &target).join("channel_groups.db").exists()); +} diff --git a/backend/repository/src/storage_const.rs b/backend/repository/src/storage_const.rs index 53363a951..153b72604 100644 --- a/backend/repository/src/storage_const.rs +++ b/backend/repository/src/storage_const.rs @@ -26,3 +26,7 @@ pub const STALKER_EPISODE_FILE: &str = "stalker_episode"; pub const STALKER_SEASONS_FILE: &str = "stalker_seasons"; pub const STALKER_SERIES_ROOTS_FILE: &str = "stalker_series_roots"; pub const STALKER_EPG_FILE: &str = "stalker_epg"; + +pub const FILE_CHANNEL_GROUPS: &str = "channel_groups.db"; +pub const FILE_CHANNEL_GROUPS_MEMBER: &str = "channel_groups.member.db"; +pub const FILE_CHANNEL_GROUPS_ALIAS: &str = "channel_groups.alias.db"; diff --git a/backend/repository/src/stream_history/mod.rs b/backend/repository/src/stream_history/mod.rs index b31c4548f..eccf97181 100644 --- a/backend/repository/src/stream_history/mod.rs +++ b/backend/repository/src/stream_history/mod.rs @@ -122,6 +122,7 @@ pub(crate) mod tests { stream_channel: shared::model::StreamChannel { target_id: 1, virtual_id: 1234, + group_virtual_id: None, provider_id: 1, input_name: shared::utils::Internable::intern("provider-input"), item_type: shared::model::PlaylistItemType::Live, diff --git a/backend/repository/src/target_stream_item.rs b/backend/repository/src/target_stream_item.rs new file mode 100644 index 000000000..cabcb6a4c --- /dev/null +++ b/backend/repository/src/target_stream_item.rs @@ -0,0 +1,423 @@ +use crate::{ChannelGroupRecord, PlaylistStorageState}; +use shared::{ + model::{M3uPlaylistItem, VirtualId, XtreamPlaylistItem}, + utils::Internable, +}; +use std::sync::Arc; +use tuliprox_core::model::{AppConfig, ConfigTarget}; + +#[derive(Debug, Clone, PartialEq)] +pub enum ResolvedStreamTarget { + Item, + Group(Arc), + Member { group: Arc, index: usize }, +} + +pub async fn classify_virtual_id( + app_config: &Arc, + playlists: &PlaylistStorageState, + target: &ConfigTarget, + vid: VirtualId, +) -> ResolvedStreamTarget { + let target_data = playlists.channel_groups.get_or_load(app_config, playlists, target).await; + if let Some(group) = target_data.groups.get(&vid) { + return ResolvedStreamTarget::Group(Arc::clone(group)); + } + if let Some(group_vid) = target_data.aliases.get(&vid) { + if let Some(group) = target_data.groups.get(group_vid) { + return ResolvedStreamTarget::Group(Arc::clone(group)); + } + } + if let Some((group_vid, index)) = target_data.members.get(&vid) { + if let Some(group) = target_data.groups.get(group_vid) { + return ResolvedStreamTarget::Member { group: Arc::clone(group), index: *index }; + } + } + ResolvedStreamTarget::Item +} + +#[must_use] +pub fn member_to_xtream_item(group: &ChannelGroupRecord, index: usize, target: &ConfigTarget) -> XtreamPlaylistItem { + let _ = target; + let member = &group.members[index]; + XtreamPlaylistItem { + virtual_id: member.member_vid, + provider_id: member.provider_id, + name: Arc::clone(&member.title), + logo: "".intern(), + logo_small: "".intern(), + group: Arc::clone(&group.key), + title: Arc::clone(&member.title), + parent_code: "".intern(), + rec: "".intern(), + url: Arc::clone(&member.url), + epg_channel_id: member.epg_channel_id.as_ref().map(Arc::clone), + xtream_cluster: member.cluster, + additional_properties: member.additional_properties.clone(), + item_type: member.item_type, + category_id: 0, + input_name: Arc::clone(&member.input_name), + channel_no: 0, + source_ordinal: 0, + input_stream_id: Arc::clone(&member.input_stream_id), + upstream_user_agent: None, + } +} + +#[must_use] +pub fn member_to_m3u_item(group: &ChannelGroupRecord, index: usize, target: &ConfigTarget) -> M3uPlaylistItem { + let _ = target; + let member = &group.members[index]; + M3uPlaylistItem { + virtual_id: member.member_vid, + provider_id: member.provider_id.to_string().intern(), + name: Arc::clone(&member.title), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: Arc::clone(&group.key), + title: Arc::clone(&member.title), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: Arc::clone(&member.url), + epg_channel_id: member.epg_channel_id.as_ref().map(Arc::clone), + input_name: Arc::clone(&member.input_name), + item_type: member.item_type, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: None, + input_stream_id: Arc::clone(&member.input_stream_id), + upstream_user_agent: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + persist_playlist_views_with_groups, ChannelGroupOrigin, ChannelGroupStore, PendingChannelGroup, + PlaylistStorageState, TargetPlaylistPersistOptions, + }; + use arc_swap::ArcSwap; + use shared::model::{ + ChannelGroupPolicy, ConfigPaths, ConfigTargetDto, M3uTargetOutputDto, PlaylistGroup, PlaylistItem, + PlaylistItemHeader, PlaylistItemType, TargetOutputDto, UUIDType, XtreamCluster, + }; + use std::path::PathBuf; + use tuliprox_core::model::{AppConfig, Config, SourcesConfig}; + + struct PersistedGroupEnv { + app_config: Arc, + playlists: Arc, + target: ConfigTarget, + _dir: tempfile::TempDir, + group_vid: VirtualId, + plain_vid: VirtualId, + member_vids: Vec, + favourite_vid: VirtualId, + } + + impl PersistedGroupEnv { + async fn republish_without_groups(&mut self) { + let (mut plain, _) = grouped_fixture_ungrouped(); + persist_playlist_views_with_groups( + &self.app_config, + &mut plain, + None, + None, + &self.target, + Some(&self.playlists), + TargetPlaylistPersistOptions::default(), + &mut [], + ) + .await + .expect("republished"); + } + } + + fn test_app_config(storage_dir: &std::path::Path) -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + storage_dir: storage_dir.to_string_lossy().into_owned(), + ..Config::default() + })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(arc_swap::ArcSwapOption::default()), + api_proxy: Arc::new(arc_swap::ArcSwapOption::default()), + file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(arc_swap::ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::new()), + }) + } + + fn target_path(app_config: &AppConfig, target: &ConfigTarget) -> PathBuf { + let config = app_config.config.load(); + crate::get_target_storage_path(&config, &target.name).expect("target storage path") + } + + fn grouped_fixture() -> (Vec, Vec) { + let group_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000001"); + let group_channel = PlaylistItem { + header: PlaylistItemHeader { + uuid: group_uuid, + id: "bbc-one".intern(), + input_stream_id: "bbc-one".intern(), + name: "BBC One".intern(), + title: "BBC One".intern(), + group: "Entertainment".intern(), + url: "http://korath/live/101".intern(), + input_name: "korath".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + channel_group_uuid: Some(group_uuid), + ..PlaylistItemHeader::default() + }, + }; + let normal_channel = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000002"), + id: "cnn".intern(), + input_stream_id: "cnn".intern(), + name: "CNN".intern(), + title: "CNN".intern(), + group: "News".intern(), + url: "http://korath/live/cnn".intern(), + input_name: "korath".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let member1 = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000011"), + id: "201".intern(), + input_stream_id: "201".intern(), + name: "BBC One HD".intern(), + title: "BBC One HD".intern(), + group: "Entertainment".intern(), + url: "http://xandar/live/201".intern(), + input_name: "xandar".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let member2 = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000012"), + id: "101".intern(), + input_stream_id: "101".intern(), + name: "BBC One FHD".intern(), + title: "BBC One FHD".intern(), + group: "Entertainment".intern(), + url: "http://korath/live/101".intern(), + input_name: "korath".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + let playlist = vec![ + PlaylistGroup { + id: 1, + title: "Entertainment".intern(), + channels: vec![group_channel], + xtream_cluster: XtreamCluster::Live, + }, + PlaylistGroup { + id: 2, + title: "News".intern(), + channels: vec![normal_channel], + xtream_cluster: XtreamCluster::Live, + }, + ]; + let pending = vec![PendingChannelGroup { + group_uuid, + key: Arc::from("bbc one"), + origin: ChannelGroupOrigin::Auto, + policy: ChannelGroupPolicy::Priority, + members: vec![member1, member2], + orphaned: vec![], + epg_split: false, + }]; + (playlist, pending) + } + + fn grouped_fixture_ungrouped() -> (Vec, Vec) { + let (mut playlist, _) = grouped_fixture(); + playlist.remove(0); + (playlist, Vec::new()) + } + + fn grouped_fixture_with_favourite_copy() -> (Vec, Vec) { + let (mut playlist, pending) = grouped_fixture(); + let group_item = &playlist[0].channels[0]; + let fav_copy = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000099"), + channel_group_uuid: Some(group_item.header.uuid), + group: "My Favourites".intern(), + ..group_item.header.clone() + }, + }; + playlist.push(PlaylistGroup { + id: 99, + title: "My Favourites".intern(), + channels: vec![fav_copy], + xtream_cluster: XtreamCluster::Live, + }); + (playlist, pending) + } + + async fn persisted_group_env() -> PersistedGroupEnv { + let dir = tempfile::tempdir().expect("tempdir"); + let app_config = test_app_config(dir.path()); + let playlists = Arc::new(PlaylistStorageState::new()); + let target = ConfigTarget::from(&ConfigTargetDto { + name: "target-group-env".to_string(), + output: vec![TargetOutputDto::M3u(M3uTargetOutputDto { + filename: Some("playlist.m3u".to_string()), + ..M3uTargetOutputDto::default() + })], + ..ConfigTargetDto::default() + }); + let (mut playlist, mut pending) = grouped_fixture(); + persist_playlist_views_with_groups( + &app_config, + &mut playlist, + None, + None, + &target, + Some(&playlists), + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .expect("persists"); + let group_vid = playlist[0].channels[0].header.virtual_id; + let plain_vid = playlist[1].channels[0].header.virtual_id; + let path = target_path(&app_config, &target); + let record = ChannelGroupStore::read_group(&path, group_vid).expect("read").expect("group"); + let member_vids = record.members.iter().map(|m| m.member_vid).collect(); + PersistedGroupEnv { + app_config, + playlists, + target, + _dir: dir, + group_vid, + plain_vid, + member_vids, + favourite_vid: VirtualId::default(), + } + } + + async fn persisted_group_env_with_favourite() -> PersistedGroupEnv { + let dir = tempfile::tempdir().expect("tempdir"); + let app_config = test_app_config(dir.path()); + let playlists = Arc::new(PlaylistStorageState::new()); + let target = ConfigTarget::from(&ConfigTargetDto { + name: "target-fav-env".to_string(), + output: vec![TargetOutputDto::M3u(M3uTargetOutputDto { + filename: Some("playlist.m3u".to_string()), + ..M3uTargetOutputDto::default() + })], + ..ConfigTargetDto::default() + }); + let (mut playlist, mut pending) = grouped_fixture_with_favourite_copy(); + persist_playlist_views_with_groups( + &app_config, + &mut playlist, + None, + None, + &target, + Some(&playlists), + TargetPlaylistPersistOptions::default(), + &mut pending, + ) + .await + .expect("persists"); + let group_vid = playlist[0].channels[0].header.virtual_id; + let plain_vid = playlist[1].channels[0].header.virtual_id; + let favourite_vid = + playlist.iter().find(|g| g.title.as_ref() == "My Favourites").unwrap().channels[0].header.virtual_id; + let path = target_path(&app_config, &target); + let record = ChannelGroupStore::read_group(&path, group_vid).expect("read").expect("group"); + let member_vids = record.members.iter().map(|m| m.member_vid).collect(); + PersistedGroupEnv { app_config, playlists, target, _dir: dir, group_vid, plain_vid, member_vids, favourite_vid } + } + + #[tokio::test] + async fn classify_distinguishes_item_group_and_member() { + let env = persisted_group_env().await; + assert!(matches!( + classify_virtual_id(&env.app_config, &env.playlists, &env.target, env.plain_vid).await, + ResolvedStreamTarget::Item + )); + assert!(matches!( + classify_virtual_id(&env.app_config, &env.playlists, &env.target, env.group_vid).await, + ResolvedStreamTarget::Group(_) + )); + match classify_virtual_id(&env.app_config, &env.playlists, &env.target, env.member_vids[1]).await { + ResolvedStreamTarget::Member { group, index } => { + assert_eq!(group.group_vid, env.group_vid); + assert_eq!(index, 1); + } + other => panic!("expected member, got {other:?}"), + } + } + + #[tokio::test] + async fn favourite_alias_vid_resolves_to_group() { + let env = persisted_group_env_with_favourite().await; + match classify_virtual_id(&env.app_config, &env.playlists, &env.target, env.favourite_vid).await { + ResolvedStreamTarget::Group(group) => assert_eq!(group.group_vid, env.group_vid), + other => panic!("expected group, got {other:?}"), + } + } + + #[tokio::test] + async fn member_item_carries_member_input_and_url() { + let env = persisted_group_env().await; + let ResolvedStreamTarget::Group(group) = + classify_virtual_id(&env.app_config, &env.playlists, &env.target, env.group_vid).await + else { + panic!() + }; + let item = member_to_xtream_item(&group, 1, &env.target); + assert_eq!(item.virtual_id, env.member_vids[1]); + assert_eq!(item.input_name.as_ref(), "korath"); + assert!(item.url.contains("korath")); + } + + #[tokio::test] + async fn cache_reloads_after_republish() { + let mut env = persisted_group_env().await; + let _ = classify_virtual_id(&env.app_config, &env.playlists, &env.target, env.group_vid).await; + env.republish_without_groups().await; + assert!(matches!( + classify_virtual_id(&env.app_config, &env.playlists, &env.target, env.group_vid).await, + ResolvedStreamTarget::Item + )); + } +} diff --git a/backend/session/Cargo.toml b/backend/session/Cargo.toml index 3ae7df636..159fc17fc 100644 --- a/backend/session/Cargo.toml +++ b/backend/session/Cargo.toml @@ -24,6 +24,8 @@ jsonwebtoken = { version = "10.4.0", default-features = false, features = ["aws_ log.workspace = true rand = "0.9.5" reqwest = { version = "0.13.4", features = ["json", "stream", "rustls", "socks", "form", "query"] } +serde.workspace = true +serde_json.workspace = true thiserror.workspace = true lru = "0.16.4" tokio = { workspace = true, features = ["rt-multi-thread", "fs", "time", "macros", "sync"] } diff --git a/backend/session/src/active_provider_manager.rs b/backend/session/src/active_provider_manager.rs index dbc16d9b9..5ea1e5e25 100644 --- a/backend/session/src/active_provider_manager.rs +++ b/backend/session/src/active_provider_manager.rs @@ -22,8 +22,8 @@ const EVICTED_PROVIDER_RELEASE_POLL_INTERVAL: Duration = Duration::from_millis(1 #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct ProviderReleaseSnapshot { pub addr: SocketAddr, - single_allocations: Vec, - shared_subscribers: Vec, + pub(crate) single_allocations: Vec, + pub(crate) shared_subscribers: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -97,6 +97,7 @@ mod tests; mod account_health; mod allocation; mod capacity; +mod capacity_probe; mod leases; mod preemption; mod release; @@ -109,3 +110,4 @@ use self::{ release::{ProviderAllocationGuard, ReleaseAction}, shared::{SharedConnectionId, SharedConnections}, }; +pub use capacity_probe::*; diff --git a/backend/session/src/active_provider_manager/allocation.rs b/backend/session/src/active_provider_manager/allocation.rs index 91bc3db2d..a50a4ffed 100644 --- a/backend/session/src/active_provider_manager/allocation.rs +++ b/backend/session/src/active_provider_manager/allocation.rs @@ -32,6 +32,10 @@ use tuliprox_core::{ impl ProviderReleaseSnapshot { pub fn is_empty(&self) -> bool { self.single_allocations.is_empty() && self.shared_subscribers.is_empty() } + + pub(crate) fn from_single_allocations(addr: SocketAddr, single_allocations: Vec) -> Self { + Self { addr, single_allocations, shared_subscribers: Vec::new() } + } } /// Resolves a public live HLS session token to its stable provider playback owner. diff --git a/backend/session/src/active_provider_manager/capacity_probe.rs b/backend/session/src/active_provider_manager/capacity_probe.rs new file mode 100644 index 000000000..f5b5c571f --- /dev/null +++ b/backend/session/src/active_provider_manager/capacity_probe.rs @@ -0,0 +1,145 @@ +use super::{AcquireProviderParams, ActiveProviderManager, ConnectionKind, OwnerProviderPin, PlaybackLeaseRef}; +pub use shared::model::CapacityTier; +use std::{ + collections::HashSet, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, +}; +use tuliprox_core::model::{AllocationId, ProviderAllocation, ProviderHandle}; + +impl ActiveProviderManager { + pub(super) fn probe_connection_candidate( + &self, + provider_or_input_name: &Arc, + allow_grace: bool, + session_owner: Option<&str>, + ) -> ProviderAllocation { + if let Some(owner) = session_owner { + match self.owner_provider_pin(provider_or_input_name, owner) { + Some(OwnerProviderPin::Reserved(reserved_provider)) => { + let allocation = + self.providers.probe_exact_connection_with_grace_override(&reserved_provider, allow_grace); + if matches!(allocation, ProviderAllocation::Exhausted) + || self.exceeds_reserved_capacity(&reserved_provider, Some(owner)) + { + return ProviderAllocation::Exhausted; + } + return allocation; + } + Some(OwnerProviderPin::Preferred(affinity_provider)) => { + let allocation = + self.providers.probe_exact_connection_with_grace_override(&affinity_provider, false); + if !matches!(allocation, ProviderAllocation::Exhausted) + && !self.exceeds_reserved_capacity(&affinity_provider, Some(owner)) + { + return allocation; + } + } + None => {} + } + } + + let candidate_count = self.providers.provider_names_for_input(provider_or_input_name).len(); + let attempts = candidate_count.max(1); + let mut skipped_reserved = HashSet::new(); + for _ in 0..attempts { + let allocation = self.providers.probe_connection_with_grace_override_excluding( + provider_or_input_name, + allow_grace, + &skipped_reserved, + ); + if matches!(allocation, ProviderAllocation::Exhausted) { + break; + } + if let Some(provider_name) = allocation.get_provider_name() { + if self.exceeds_reserved_capacity(&provider_name, session_owner) { + skipped_reserved.insert(provider_name); + if skipped_reserved.len() >= attempts { + break; + } + continue; + } + } + return allocation; + } + + ProviderAllocation::Exhausted + } + + pub fn capacity_probe( + &self, + input_name: &Arc, + priority: i8, + kind: ConnectionKind, + session_owner: Option<&str>, + own_releasing: &[AllocationId], + ) -> CapacityTier { + let free_probe = self.probe_connection_candidate(input_name, false, session_owner); + if matches!(free_probe, ProviderAllocation::Available(_)) { + return CapacityTier::Free; + } + + if !own_releasing.is_empty() { + let connections = self.read_connections(); + let has_own_release = own_releasing.iter().any(|alloc_id| { + if let Some(info) = connections.single.get(alloc_id) { + if let Some(prov_name) = info.allocation.get_provider_name() { + if self.providers.is_provider_for_input(&prov_name, input_name) || &prov_name == input_name { + return true; + } + } + } + if let Some(key) = connections.shared.shared_by_allocation_id.get(alloc_id) { + if let Some(shared) = connections.shared.by_key.get(key) { + if let Some(prov_name) = shared.allocation.get_provider_name() { + if self.providers.is_provider_for_input(&prov_name, input_name) || &prov_name == input_name + { + return true; + } + } + } + } + false + }); + if has_own_release { + return CapacityTier::OwnRelease; + } + } + + let grace_probe = self.probe_connection_candidate(input_name, true, session_owner); + if matches!(grace_probe, ProviderAllocation::Available(_) | ProviderAllocation::GracePeriod(_)) { + return CapacityTier::Grace; + } + + let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); + let has_preempt_candidate = { + let connections = self.read_connections(); + self.select_preemption_candidate(&connections, input_name, priority, kind, &reserved_providers).is_some() + }; + if has_preempt_candidate { + return CapacityTier::Preempt; + } + + CapacityTier::Exhausted + } + + pub fn try_acquire_connection_no_preempt( + &self, + input_name: &Arc, + allow_grace: bool, + addr: &SocketAddr, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let params = AcquireProviderParams { addr, priority, kind, lease }; + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) + } +} diff --git a/backend/session/src/active_provider_manager/tests.rs b/backend/session/src/active_provider_manager/tests.rs index 2b030caa3..3bb57e446 100644 --- a/backend/session/src/active_provider_manager/tests.rs +++ b/backend/session/src/active_provider_manager/tests.rs @@ -2,6 +2,7 @@ use super::{ActiveProviderManager, ConnectionKind, PlaybackLeaseRef}; mod admission; mod behavior; +mod capacity_probe; mod lifecycle; mod retry; mod support; diff --git a/backend/session/src/active_provider_manager/tests/capacity_probe.rs b/backend/session/src/active_provider_manager/tests/capacity_probe.rs new file mode 100644 index 000000000..ad608ae78 --- /dev/null +++ b/backend/session/src/active_provider_manager/tests/capacity_probe.rs @@ -0,0 +1,105 @@ +use super::super::{ActiveProviderManager, CapacityTier, ConnectionKind}; +use crate::EventManager; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::{ + model::{ConfigPaths, InputType, ReverseProxyConfigDto, StreamConfigDto}, + utils::Internable, +}; +use std::{net::SocketAddr, sync::Arc}; +use tuliprox_core::{ + model::{AppConfig, Config, ConfigInput, MediaToolCapabilities, ReverseProxyConfig, SourcesConfig}, + utils::FileLockManager, +}; + +fn addr(port: u16) -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], port)) } + +fn app_config_with_sources_and_grace(sources: SourcesConfig, grace_period_millis: u64) -> AppConfig { + let mut config = Config::default(); + let dto = ReverseProxyConfigDto { + stream: Some(StreamConfigDto { grace_period_millis, ..StreamConfigDto::default() }), + ..ReverseProxyConfigDto::default() + }; + config.reverse_proxy = Some(ReverseProxyConfig::from(&dto)); + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(config)), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +fn manager_with_inputs(inputs: &[(&str, u16)]) -> ActiveProviderManager { + let sources = SourcesConfig { + inputs: inputs + .iter() + .enumerate() + .map(|(i, (name, max))| { + Arc::new(ConfigInput { + id: u16::try_from(i + 1).unwrap(), + name: (*name).intern(), + input_type: InputType::Xtream, + url: format!("http://{name}.invalid"), + username: Some("u".into()), + password: Some("p".into()), + enabled: true, + max_connections: *max, + ..ConfigInput::default() + }) + }) + .collect(), + ..SourcesConfig::default() + }; + ActiveProviderManager::new(&app_config_with_sources_and_grace(sources, 2_000), &Arc::new(EventManager::new())) +} + +#[test] +fn probe_reports_tiers_without_allocating() { + let manager = manager_with_inputs(&[("solis", 1)]); + let solis: Arc = "solis".into(); + assert_eq!(manager.capacity_probe(&solis, 0, ConnectionKind::Normal, None, &[]), CapacityTier::Free); + let held = manager.acquire_connection(&solis, &addr(1), -1, ConnectionKind::Normal).expect("slot"); + assert_eq!(manager.capacity_probe(&solis, 0, ConnectionKind::Normal, None, &[]), CapacityTier::Grace); + let total: usize = manager.active_connections().unwrap_or_default().values().sum(); + assert_eq!(total, 1); + drop(held); +} + +#[test] +fn own_pending_release_is_reported_as_own_release() { + let manager = manager_with_inputs(&[("solis", 1)]); + let solis: Arc = "solis".into(); + let held = manager.acquire_connection(&solis, &addr(1), 0, ConnectionKind::Normal).expect("slot"); + let own = [held.allocation_id]; + assert_eq!(manager.capacity_probe(&solis, 0, ConnectionKind::Normal, None, &own), CapacityTier::OwnRelease); +} + +#[test] +fn higher_priority_sees_preempt_when_grace_is_used() { + let manager = manager_with_inputs(&[("solis", 1)]); + let solis: Arc = "solis".into(); + let _low = manager.acquire_connection(&solis, &addr(1), 10, ConnectionKind::Normal).expect("slot"); + let _grace = manager + .try_acquire_connection_no_preempt(&solis, true, &addr(2), 10, ConnectionKind::Normal, None) + .expect("grace"); + assert_eq!(manager.capacity_probe(&solis, -10, ConnectionKind::Normal, None, &[]), CapacityTier::Preempt); + assert_eq!(manager.capacity_probe(&solis, 10, ConnectionKind::Normal, None, &[]), CapacityTier::Exhausted); +} diff --git a/backend/session/src/active_user_manager/activity.rs b/backend/session/src/active_user_manager/activity.rs index 75f739a85..d05c26d18 100644 --- a/backend/session/src/active_user_manager/activity.rs +++ b/backend/session/src/active_user_manager/activity.rs @@ -1,7 +1,7 @@ use super::{ next_session_incarnation, remember_session_addr, url_host_key, ActiveUserManager, CreateUserSessionParams, - PlaybackLifecycle, SessionChangeSignal, SessionIdentity, SocketRegistration, UserConnectionData, UserSession, - UserSessionParams, + MemberBinding, PlaybackLifecycle, SessionChangeSignal, SessionIdentity, SocketRegistration, UserConnectionData, + UserSession, UserSessionParams, }; use shared::{ model::UserConnectionPermission, @@ -40,6 +40,7 @@ impl ActiveUserManager { permission: params.connection_permission, connection_kind: params.connection_kind, lifecycle: PlaybackLifecycle::Prepared, + member_binding: None, } } @@ -304,4 +305,14 @@ impl ActiveUserManager { Some(connection_data.sessions[session_index].clone()) } + + pub async fn bind_member(&self, username: &str, token: &str, binding: MemberBinding) { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + if let Some(session) = connection_data.sessions.iter_mut().find(|s| s.token == token) { + session.member_binding = Some(binding); + } + } } diff --git a/backend/session/src/active_user_manager/admission.rs b/backend/session/src/active_user_manager/admission.rs index 138ca4cf5..5703e56ed 100644 --- a/backend/session/src/active_user_manager/admission.rs +++ b/backend/session/src/active_user_manager/admission.rs @@ -8,6 +8,7 @@ use jsonwebtoken::get_current_timestamp; use log::debug; use shared::model::{StreamInfo, UserConnectionPermission}; use std::{collections::HashSet, net::SocketAddr, sync::atomic::Ordering}; +use tuliprox_core::model::AllocationId; /// Panel identity of a removed stream row, as carried by `DisconnectedStream`. pub(super) type RemovedStreamKey = (SocketAddr, u32); @@ -185,6 +186,21 @@ impl UserConnectionData { } impl ActiveUserManager { + pub async fn pending_release_allocations(&self, username: &str) -> Vec { + self.pending_provider_release(username).await.map_or_else(Vec::new, |snapshot| snapshot.single_allocations) + } + + pub async fn set_pending_release_for_test(&self, username: &str, allocations: &[AllocationId]) { + self.set_pending_provider_release( + username, + crate::active_provider_manager::ProviderReleaseSnapshot::from_single_allocations( + SocketAddr::from(([127, 0, 0, 1], 0)), + allocations.to_vec(), + ), + ) + .await; + } + pub(crate) async fn pending_provider_release(&self, username: &str) -> Option { self.pending_provider_releases.lock().await.get(username).cloned() } diff --git a/backend/session/src/active_user_manager/mod.rs b/backend/session/src/active_user_manager/mod.rs index 97149690a..d91d17960 100644 --- a/backend/session/src/active_user_manager/mod.rs +++ b/backend/session/src/active_user_manager/mod.rs @@ -111,6 +111,14 @@ pub enum SessionProviderHeaders { Headers(HashMap), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MemberBinding { + pub group_vid: u32, + pub member_vid: u32, + pub member_index: usize, + pub input_name: Arc, +} + #[derive(Clone, Debug)] pub struct UserSession { pub token: String, @@ -140,6 +148,7 @@ pub struct UserSession { pub permission: UserConnectionPermission, pub connection_kind: Option, pub lifecycle: PlaybackLifecycle, + pub member_binding: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/backend/session/src/active_user_manager/observability.rs b/backend/session/src/active_user_manager/observability.rs index a3326734c..5538a7f95 100644 --- a/backend/session/src/active_user_manager/observability.rs +++ b/backend/session/src/active_user_manager/observability.rs @@ -79,6 +79,26 @@ impl ActiveUserManager { .collect() } + /// Collect active session virtual ids for a user. + pub async fn active_session_vids(&self, username: &str) -> Vec { + let connections = self.connections.read().await; + let Some(data) = connections.by_key.get(username) else { + return Vec::new(); + }; + if data.sessions.is_empty() { + data.streams.iter().filter(|s| !s.preserved).map(|s| s.channel.virtual_id).collect() + } else { + data.sessions.iter().map(|s| s.virtual_id).collect() + } + } + + /// Read bound member input name for a user session if any. + pub async fn bound_member_input(&self, username: &str) -> Option> { + let connections = self.connections.read().await; + let data = connections.by_key.get(username)?; + data.sessions.iter().find_map(|s| s.member_binding.as_ref().map(|b| Arc::clone(&b.input_name))) + } + pub(super) async fn log_active_user(&self) { let is_log_user_enabled = self.is_log_user_enabled(); let (user_count, user_connection_count) = { self.active_users_and_connections().await }; diff --git a/backend/session/src/active_user_manager/session_lifecycle.rs b/backend/session/src/active_user_manager/session_lifecycle.rs index e5bfc03fd..32aae6934 100644 --- a/backend/session/src/active_user_manager/session_lifecycle.rs +++ b/backend/session/src/active_user_manager/session_lifecycle.rs @@ -84,6 +84,7 @@ impl Default for UserSession { permission: UserConnectionPermission::Allowed, connection_kind: None, lifecycle: PlaybackLifecycle::default(), + member_binding: None, } } } diff --git a/backend/session/src/active_user_manager/tests/observability.rs b/backend/session/src/active_user_manager/tests/observability.rs index 04bd223c4..1a07f16ce 100644 --- a/backend/session/src/active_user_manager/tests/observability.rs +++ b/backend/session/src/active_user_manager/tests/observability.rs @@ -218,6 +218,7 @@ async fn check_divergence_detects_stream_without_counted_session() { epg_channel_id: None, epg_reference_ts: None, upstream_user_agent: None, + group_virtual_id: None, }, user_agent: "ua".to_string(), country_code: None, diff --git a/backend/session/src/active_user_manager/tests/support.rs b/backend/session/src/active_user_manager/tests/support.rs index ac32cba02..9c6521f97 100644 --- a/backend/session/src/active_user_manager/tests/support.rs +++ b/backend/session/src/active_user_manager/tests/support.rs @@ -30,6 +30,7 @@ pub(in crate::active_user_manager::tests) fn test_channel(virtual_id: u32) -> St epg_channel_id: None, epg_reference_ts: None, upstream_user_agent: None, + group_virtual_id: None, } } @@ -51,6 +52,7 @@ pub(in crate::active_user_manager::tests) fn test_adaptive_channel(virtual_id: u epg_channel_id: None, epg_reference_ts: None, upstream_user_agent: None, + group_virtual_id: None, } } diff --git a/backend/session/src/channel_group/circuit.rs b/backend/session/src/channel_group/circuit.rs new file mode 100644 index 000000000..3adc15090 --- /dev/null +++ b/backend/session/src/channel_group/circuit.rs @@ -0,0 +1,299 @@ +use axum::http::StatusCode; +use dashmap::DashMap; +use shared::model::ChannelGroupSelectionConfig; +use std::{ + collections::{HashSet, VecDeque}, + sync::Arc, + time::{Duration, Instant}, +}; + +pub type MemberKey = (Arc, Arc); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailureClass { + Stream, + Transport, + Auth, + Capacity, +} + +impl FailureClass { + pub fn from_status(status: StatusCode) -> Self { + match status.as_u16() { + 401 | 403 => Self::Auth, + 429 | 509 => Self::Capacity, + _ => Self::Stream, + } + } + + pub fn opens_member_circuit(self) -> bool { !matches!(self, Self::Capacity) } + + pub fn counts_for_input(self) -> bool { matches!(self, Self::Transport | Self::Auth) } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CircuitView { + Closed, + Counting(u8), + Open { remaining: Duration }, + HalfOpenInFlight, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CircuitPolicy { + pub failures: u8, + pub cooldown: Duration, + pub cooldown_max: Duration, + pub input_failures: u8, + pub input_window: Duration, +} + +impl CircuitPolicy { + pub fn new(failures: u8, cooldown: Duration, cooldown_max: Duration) -> Self { + Self { failures, cooldown, cooldown_max, input_failures: 3, input_window: Duration::from_secs(120) } + } + + pub fn with_input(mut self, input_failures: u8, input_window: Duration) -> Self { + self.input_failures = input_failures; + self.input_window = input_window; + self + } +} + +impl Default for CircuitPolicy { + fn default() -> Self { Self::new(3, Duration::from_secs(60), Duration::from_mins(15)) } +} + +impl From<&ChannelGroupSelectionConfig> for CircuitPolicy { + fn from(cfg: &ChannelGroupSelectionConfig) -> Self { + Self { + failures: cfg.circuit_failures, + cooldown: Duration::from_secs(cfg.circuit_cooldown_secs), + cooldown_max: Duration::from_secs(cfg.circuit_cooldown_max_secs), + input_failures: cfg.input_circuit_failures, + input_window: Duration::from_secs(cfg.input_failure_window_secs), + } + } +} + +pub use shared::model::CircuitState; + +#[derive(Debug, Clone)] +pub struct CircuitEntry { + pub failures: u8, + pub open_level: u8, + pub state: CircuitState, +} + +impl Default for CircuitEntry { + fn default() -> Self { Self { failures: 0, open_level: 0, state: CircuitState::Closed } } +} + +#[derive(Debug, Clone, Default)] +pub struct InputCircuitEntry { + pub circuit: CircuitEntry, + pub recent: VecDeque<(MemberKey, Instant)>, +} + +#[derive(Default)] +pub struct CircuitSharedState { + pub circuits: DashMap, + pub input_circuits: DashMap, InputCircuitEntry>, +} + +pub struct HalfOpenGuard { + pub(crate) member_key: MemberKey, + pub(crate) input_name: Option>, + pub(crate) admit_time: Instant, + pub(crate) shared: Arc, + pub(crate) consumed: bool, +} + +impl Drop for HalfOpenGuard { + fn drop(&mut self) { + if !self.consumed { + if let Some(mut entry) = self.shared.circuits.get_mut(&self.member_key) { + if entry.state == CircuitState::HalfOpenInFlight { + entry.state = CircuitState::OpenUntil(self.admit_time); + } + } + if let Some(input) = &self.input_name { + if let Some(mut entry) = self.shared.input_circuits.get_mut(input) { + if entry.circuit.state == CircuitState::HalfOpenInFlight { + entry.circuit.state = CircuitState::OpenUntil(self.admit_time); + } + } + } + } + } +} + +pub enum CircuitAdmission { + Closed, + HalfOpen(HalfOpenGuard), + Open, +} + +pub(crate) fn compute_cooldown(policy: &CircuitPolicy, open_level: u8) -> Duration { + let multiplier = 1u32.checked_shl(u32::from(open_level.min(16))).unwrap_or(u32::MAX); + let scaled = policy.cooldown.saturating_mul(multiplier); + scaled.min(policy.cooldown_max) +} + +pub(crate) fn record_input_failure_in_shared( + shared: &CircuitSharedState, + key: &MemberKey, + now: Instant, + policy: &CircuitPolicy, +) { + let mut entry = shared.input_circuits.entry(Arc::clone(&key.0)).or_default(); + entry.recent.push_back((key.clone(), now)); + let window = policy.input_window; + entry.recent.retain(|(_, ts)| if *ts <= now { now.duration_since(*ts) <= window } else { true }); + while entry.recent.len() > usize::from(policy.input_failures) * 4 { + entry.recent.pop_front(); + } + let mut distinct = HashSet::new(); + for (k, _) in &entry.recent { + distinct.insert(k); + if distinct.len() >= 2 { + break; + } + } + if entry.recent.len() >= usize::from(policy.input_failures) + && distinct.len() >= 2 + && entry.circuit.state == CircuitState::Closed + { + let cooldown = compute_cooldown(policy, entry.circuit.open_level); + entry.circuit.state = CircuitState::OpenUntil(now + cooldown); + entry.circuit.open_level = entry.circuit.open_level.saturating_add(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::channel_group::health::MemberHealthIndex; + + fn key(input: &str, stream_id: &str) -> MemberKey { (input.into(), stream_id.into()) } + + #[test] + fn opens_after_threshold_and_backs_off_exponentially() { + let index = MemberHealthIndex::new(CircuitPolicy::new(2, Duration::from_secs(60), Duration::from_mins(15))); + let k = key("solis", "1"); + let t0 = Instant::now(); + index.record_failure(&k, t0); + assert!(matches!(index.admit(&k, t0), CircuitAdmission::Closed)); + index.record_failure(&k, t0); + assert!(matches!(index.admit(&k, t0), CircuitAdmission::Open)); + let t1 = t0 + Duration::from_secs(61); + let CircuitAdmission::HalfOpen(guard) = index.admit(&k, t1) else { panic!("expected half-open") }; + index.record_failure_with(guard, t1); + assert!(matches!(index.admit(&k, t1 + Duration::from_secs(119)), CircuitAdmission::Open)); + assert!(matches!(index.admit(&k, t1 + Duration::from_secs(121)), CircuitAdmission::HalfOpen(_))); + } + + #[test] + fn half_open_is_single_flight_under_concurrency() { + let index = MemberHealthIndex::new(CircuitPolicy::new(1, Duration::from_secs(1), Duration::from_secs(8))); + let k = key("korath", "7"); + let t0 = Instant::now(); + index.record_failure(&k, t0); + let later = t0 + Duration::from_secs(2); + let granted = std::sync::Mutex::new(Vec::new()); + std::thread::scope(|scope| { + for _ in 0..16 { + scope.spawn(|| { + if let CircuitAdmission::HalfOpen(guard) = index.admit(&k, later) { + granted.lock().expect("lock").push(guard); + } + }); + } + }); + assert_eq!(granted.lock().expect("lock").len(), 1); + } + + #[test] + fn dropped_half_open_guard_allows_next_probe() { + let index = MemberHealthIndex::new(CircuitPolicy::new(1, Duration::from_secs(1), Duration::from_secs(8))); + let k = key("veyra", "3"); + let t0 = Instant::now(); + index.record_failure(&k, t0); + let later = t0 + Duration::from_secs(2); + drop(index.admit(&k, later)); + assert!(matches!(index.admit(&k, later), CircuitAdmission::HalfOpen(_))); + } + + #[test] + fn success_closes_and_resets_backoff() { + let index = MemberHealthIndex::new(CircuitPolicy::new(1, Duration::from_secs(1), Duration::from_secs(8))); + let k = key("orun", "4"); + let t0 = Instant::now(); + index.record_failure(&k, t0); + let CircuitAdmission::HalfOpen(guard) = index.admit(&k, t0 + Duration::from_secs(2)) else { panic!() }; + index.record_success_with(guard); + assert!(matches!(index.admit(&k, t0 + Duration::from_secs(2)), CircuitAdmission::Closed)); + assert!(matches!(index.circuit_view(&k, t0 + Duration::from_secs(2)), CircuitView::Closed)); + } + + #[test] + fn m3u_members_without_provider_id_do_not_share_a_circuit() { + let index = MemberHealthIndex::new(CircuitPolicy::new(1, Duration::from_secs(60), Duration::from_secs(60))); + let t0 = Instant::now(); + index.record_failure(&key("m3u-in", "chan-a"), t0); + assert!(matches!(index.admit(&key("m3u-in", "chan-a"), t0), CircuitAdmission::Open)); + assert!(matches!(index.admit(&key("m3u-in", "chan-b"), t0), CircuitAdmission::Closed)); + } + + #[test] + fn input_circuit_needs_two_distinct_members() { + let index = MemberHealthIndex::new( + CircuitPolicy::new(5, Duration::from_secs(60), Duration::from_secs(60)) + .with_input(3, Duration::from_secs(120)), + ); + let t0 = Instant::now(); + for _ in 0..3 { + index.record_classified_failure(&key("solis", "1"), FailureClass::Transport, t0); + } + assert!(matches!(index.input_circuit_view("solis", t0), CircuitView::Closed | CircuitView::Counting(_))); + index.record_classified_failure(&key("solis", "2"), FailureClass::Auth, t0); + assert!(matches!(index.input_circuit_view("solis", t0), CircuitView::Open { .. })); + assert!(matches!(index.admit(&key("solis", "77"), t0), CircuitAdmission::Open)); + assert!(matches!(index.admit(&key("korath", "1"), t0), CircuitAdmission::Closed)); + } + + #[test] + fn input_failures_outside_window_do_not_count() { + let index = MemberHealthIndex::new( + CircuitPolicy::new(5, Duration::from_secs(60), Duration::from_secs(60)) + .with_input(2, Duration::from_secs(120)), + ); + let t0 = Instant::now(); + index.record_classified_failure(&key("solis", "1"), FailureClass::Transport, t0); + index.record_classified_failure(&key("solis", "2"), FailureClass::Transport, t0 + Duration::from_secs(121)); + assert!(!matches!(index.input_circuit_view("solis", t0 + Duration::from_secs(121)), CircuitView::Open { .. })); + } + + #[test] + fn capacity_failure_never_opens_a_circuit() { + let index = MemberHealthIndex::new( + CircuitPolicy::new(1, Duration::from_secs(60), Duration::from_secs(60)) + .with_input(2, Duration::from_secs(120)), + ); + let t0 = Instant::now(); + index.record_classified_failure(&key("solis", "1"), FailureClass::Capacity, t0); + index.record_classified_failure(&key("solis", "2"), FailureClass::Capacity, t0); + assert!(matches!(index.admit(&key("solis", "1"), t0), CircuitAdmission::Closed)); + assert!(matches!(index.input_circuit_view("solis", t0), CircuitView::Closed)); + } + + #[test] + fn failure_class_from_status() { + assert_eq!(FailureClass::from_status(StatusCode::UNAUTHORIZED), FailureClass::Auth); + assert_eq!(FailureClass::from_status(StatusCode::FORBIDDEN), FailureClass::Auth); + assert_eq!(FailureClass::from_status(StatusCode::TOO_MANY_REQUESTS), FailureClass::Capacity); + assert_eq!(FailureClass::from_status(StatusCode::from_u16(509).unwrap()), FailureClass::Capacity); + assert_eq!(FailureClass::from_status(StatusCode::NOT_FOUND), FailureClass::Stream); + assert_eq!(FailureClass::from_status(StatusCode::SERVICE_UNAVAILABLE), FailureClass::Stream); + } +} diff --git a/backend/session/src/channel_group/health.rs b/backend/session/src/channel_group/health.rs new file mode 100644 index 000000000..9c9769140 --- /dev/null +++ b/backend/session/src/channel_group/health.rs @@ -0,0 +1,525 @@ +use super::circuit::{ + compute_cooldown, record_input_failure_in_shared, CircuitAdmission, CircuitPolicy, CircuitSharedState, + CircuitState, CircuitView, FailureClass, HalfOpenGuard, MemberKey, +}; +use arc_swap::ArcSwap; +use shared::model::{AudioChannels, AudioCodec, MediaQuality, VideoCodec, VideoResolution}; +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, +}; +use tuliprox_repository::GroupMember; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum QosKey { + Provider { input: Arc, provider_id: u32 }, + TargetMember { target: Arc, member_vid: u32 }, +} + +pub fn member_key(member: &GroupMember) -> MemberKey { + (Arc::clone(&member.input_name), Arc::clone(&member.input_stream_id)) +} + +pub fn qos_key(target_name: &Arc, member: &GroupMember) -> QosKey { + if member.provider_id > 0 { + QosKey::Provider { input: Arc::clone(&member.input_name), provider_id: member.provider_id } + } else { + QosKey::TargetMember { target: Arc::clone(target_name), member_vid: member.member_vid.get() } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct QosPart { + pub score_24h: u8, + pub score_7d: u8, + pub confidence: u8, + pub failure_streak: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProbePart { + pub resolution: VideoResolution, + pub codec: VideoCodec, + pub bitrate_kbps: u32, + pub audio_codec: AudioCodec, + pub audio_channels: AudioChannels, + pub audio_language: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MemberHealth { + pub qos: Option, + pub probe: Option, + pub circuit: CircuitView, +} + +pub struct MemberHealthIndex { + qos: ArcSwap>, + probe: ArcSwap>, + shared: Arc, + policy: ArcSwap, + event_sink: ArcSwap>>, +} + +impl Default for MemberHealthIndex { + fn default() -> Self { Self::new(CircuitPolicy::default()) } +} + +impl MemberHealthIndex { + pub fn new(policy: CircuitPolicy) -> Self { + Self { + qos: ArcSwap::from_pointee(HashMap::new()), + probe: ArcSwap::from_pointee(HashMap::new()), + shared: Arc::new(CircuitSharedState::default()), + policy: ArcSwap::from_pointee(policy), + event_sink: ArcSwap::from_pointee(None), + } + } + + pub fn set_event_sink(&self, sink: Arc) { + self.event_sink.store(Arc::new(Some(sink))); + } + + fn emit_circuit_changed(&self, input_name: Arc, input_stream_id: Arc, state: CircuitState) { + if let Some(sink) = self.event_sink.load().as_ref() { + sink.emit(shared::model::EventMessage::ChannelGroupCircuitChanged { input_name, input_stream_id, state }); + } + } + + pub fn health(&self, target_name: &Arc, member: &GroupMember, now: Instant) -> MemberHealth { + let q_key = qos_key(target_name, member); + let qos = self.qos.load().get(&q_key).copied(); + let m_key = member_key(member); + let probe = self.probe.load().get(&m_key).cloned(); + let circuit = self.circuit_view(&m_key, now); + MemberHealth { qos, probe, circuit } + } + + pub fn replace_qos(&self, entries: impl IntoIterator) { + let mut map = HashMap::new(); + for (k, v) in entries { + match map.entry(k) { + std::collections::hash_map::Entry::Vacant(e) => { + e.insert(v); + } + std::collections::hash_map::Entry::Occupied(mut e) => { + if v.confidence >= e.get().confidence { + e.insert(v); + } + } + } + } + self.qos.store(Arc::new(map)); + } + + pub fn replace_qos_merge(&self, entries: impl IntoIterator) { + let mut map = (**self.qos.load()).clone(); + for (k, v) in entries { + match map.entry(k) { + std::collections::hash_map::Entry::Vacant(e) => { + e.insert(v); + } + std::collections::hash_map::Entry::Occupied(mut e) => { + if v.confidence >= e.get().confidence { + e.insert(v); + } + } + } + } + self.qos.store(Arc::new(map)); + } + + pub fn patch_probe(&self, key: &MemberKey, video_json: Option<&str>, bitrate_kbps: u32) { + let quality = MediaQuality::from_ffprobe_info(None, video_json); + let (resolution, codec) = + quality.map_or((VideoResolution::default(), VideoCodec::default()), |q| (q.resolution, q.video_codec)); + + self.probe.rcu(|current| { + let mut map = (**current).clone(); + let (audio_codec, audio_channels, audio_language) = + map.get(key).map_or((AudioCodec::default(), AudioChannels::default(), None), |p| { + (p.audio_codec.clone(), p.audio_channels.clone(), p.audio_language.clone()) + }); + map.insert( + key.clone(), + ProbePart { + resolution: resolution.clone(), + codec: codec.clone(), + bitrate_kbps, + audio_codec, + audio_channels, + audio_language, + }, + ); + Arc::new(map) + }); + } + + pub fn patch_probe_audio(&self, key: &MemberKey, audio_json: Option<&str>) { + let quality = MediaQuality::from_ffprobe_info(audio_json, None); + let (audio_codec, audio_channels) = + quality.map_or((AudioCodec::default(), AudioChannels::default()), |q| (q.audio_codec, q.audio_channels)); + + let audio_language = audio_json.and_then(|raw| { + let val = serde_json::from_str::(raw).ok()?; + let obj = match &val { + serde_json::Value::Object(map) => Some(map), + serde_json::Value::Array(arr) => arr.first().and_then(serde_json::Value::as_object), + _ => None, + }?; + let lang = obj + .get("tags") + .and_then(|t| t.get("language").or_else(|| t.get("LANGUAGE"))) + .or_else(|| obj.get("language")) + .and_then(|l| l.as_str())?; + let lower = lang.trim().to_lowercase(); + if lower.is_empty() || lower == "und" { + None + } else { + Some(Arc::::from(lower)) + } + }); + + self.probe.rcu(|current| { + let mut map = (**current).clone(); + let (resolution, codec, bitrate_kbps) = + map.get(key).map_or((VideoResolution::default(), VideoCodec::default(), 0), |p| { + (p.resolution.clone(), p.codec.clone(), p.bitrate_kbps) + }); + map.insert( + key.clone(), + ProbePart { + resolution, + codec, + bitrate_kbps, + audio_codec: audio_codec.clone(), + audio_channels: audio_channels.clone(), + audio_language: audio_language.clone(), + }, + ); + Arc::new(map) + }); + } + + pub fn admit(&self, key: &MemberKey, now: Instant) -> CircuitAdmission { + let mut acquired_input_half_open = false; + + // 1. Check input circuit first + if let Some(mut input_entry) = self.shared.input_circuits.get_mut(&key.0) { + match input_entry.circuit.state { + CircuitState::Closed => {} + CircuitState::OpenUntil(until) => { + if now < until { + return CircuitAdmission::Open; + } + input_entry.circuit.state = CircuitState::HalfOpenInFlight; + acquired_input_half_open = true; + } + CircuitState::HalfOpenInFlight => { + return CircuitAdmission::Open; + } + } + } + + // 2. Check member circuit + let mut revert_input = false; + let admission = match self.shared.circuits.get_mut(key) { + None => { + if acquired_input_half_open { + CircuitAdmission::HalfOpen(HalfOpenGuard { + member_key: (Arc::clone(&key.0), Arc::clone(&key.1)), + input_name: Some(Arc::clone(&key.0)), + admit_time: now, + shared: Arc::clone(&self.shared), + consumed: false, + }) + } else { + CircuitAdmission::Closed + } + } + Some(mut member_entry) => match member_entry.state { + CircuitState::Closed => { + if acquired_input_half_open { + CircuitAdmission::HalfOpen(HalfOpenGuard { + member_key: (Arc::clone(&key.0), Arc::clone(&key.1)), + input_name: Some(Arc::clone(&key.0)), + admit_time: now, + shared: Arc::clone(&self.shared), + consumed: false, + }) + } else { + CircuitAdmission::Closed + } + } + CircuitState::OpenUntil(until) => { + if now < until { + revert_input = true; + CircuitAdmission::Open + } else { + member_entry.state = CircuitState::HalfOpenInFlight; + self.emit_circuit_changed( + Arc::clone(&key.0), + Arc::clone(&key.1), + CircuitState::HalfOpenInFlight, + ); + CircuitAdmission::HalfOpen(HalfOpenGuard { + member_key: (Arc::clone(&key.0), Arc::clone(&key.1)), + input_name: if acquired_input_half_open { Some(Arc::clone(&key.0)) } else { None }, + admit_time: now, + shared: Arc::clone(&self.shared), + consumed: false, + }) + } + } + CircuitState::HalfOpenInFlight => { + revert_input = true; + CircuitAdmission::Open + } + }, + }; + + if revert_input && acquired_input_half_open { + if let Some(mut input_entry) = self.shared.input_circuits.get_mut(&key.0) { + if input_entry.circuit.state == CircuitState::HalfOpenInFlight { + input_entry.circuit.state = CircuitState::OpenUntil(now); + } + } + } + + admission + } + + pub fn record_failure(&self, key: &MemberKey, now: Instant) { + self.record_classified_failure(key, FailureClass::Stream, now); + } + + pub fn record_classified_failure(&self, key: &MemberKey, class: FailureClass, now: Instant) { + let policy = self.policy.load(); + if class.opens_member_circuit() { + let mut entry = self.shared.circuits.entry(key.clone()).or_default(); + entry.failures = entry.failures.saturating_add(1); + if entry.failures >= policy.failures && entry.state == CircuitState::Closed { + let cooldown = compute_cooldown(&policy, entry.open_level); + entry.state = CircuitState::OpenUntil(now + cooldown); + entry.open_level = entry.open_level.saturating_add(1); + self.emit_circuit_changed(Arc::clone(&key.0), Arc::clone(&key.1), entry.state); + } + } + if class.counts_for_input() { + record_input_failure_in_shared(&self.shared, key, now, &policy); + } + } + + pub fn record_success(&self, key: &MemberKey) { + if let Some(mut entry) = self.shared.circuits.get_mut(key) { + let was_open = entry.state != CircuitState::Closed; + entry.failures = 0; + entry.open_level = 0; + entry.state = CircuitState::Closed; + if was_open { + self.emit_circuit_changed(Arc::clone(&key.0), Arc::clone(&key.1), CircuitState::Closed); + } + } + } + + pub fn record_failure_with(&self, guard: HalfOpenGuard, now: Instant) { + self.record_classified_failure_with(guard, FailureClass::Stream, now); + } + + pub fn record_classified_failure_with(&self, mut guard: HalfOpenGuard, class: FailureClass, now: Instant) { + guard.consumed = true; + let policy = self.policy.load(); + if class.opens_member_circuit() { + let mut entry = self.shared.circuits.entry(guard.member_key.clone()).or_default(); + let cooldown = compute_cooldown(&policy, entry.open_level); + entry.state = CircuitState::OpenUntil(now + cooldown); + entry.open_level = entry.open_level.saturating_add(1); + self.emit_circuit_changed(Arc::clone(&guard.member_key.0), Arc::clone(&guard.member_key.1), entry.state); + } + if let Some(input) = &guard.input_name { + if class.counts_for_input() { + let mut entry = self.shared.input_circuits.entry(Arc::clone(input)).or_default(); + let cooldown = compute_cooldown(&policy, entry.circuit.open_level); + entry.circuit.state = CircuitState::OpenUntil(now + cooldown); + entry.circuit.open_level = entry.circuit.open_level.saturating_add(1); + } + } + if class.counts_for_input() { + record_input_failure_in_shared(&self.shared, &guard.member_key, now, &policy); + } + } + + pub fn record_success_with(&self, mut guard: HalfOpenGuard) { + guard.consumed = true; + if let Some(mut entry) = self.shared.circuits.get_mut(&guard.member_key) { + let was_open = entry.state != CircuitState::Closed; + entry.failures = 0; + entry.open_level = 0; + entry.state = CircuitState::Closed; + if was_open { + self.emit_circuit_changed( + Arc::clone(&guard.member_key.0), + Arc::clone(&guard.member_key.1), + CircuitState::Closed, + ); + } + } + if let Some(input) = &guard.input_name { + if let Some(mut entry) = self.shared.input_circuits.get_mut(input) { + entry.circuit.failures = 0; + entry.circuit.open_level = 0; + entry.circuit.state = CircuitState::Closed; + entry.recent.clear(); + } + } + } + + pub fn circuit_view(&self, key: &MemberKey, now: Instant) -> CircuitView { + if let Some(input_entry) = self.shared.input_circuits.get(&key.0) { + match input_entry.circuit.state { + CircuitState::OpenUntil(until) => { + if now < until { + return CircuitView::Open { remaining: until.saturating_duration_since(now) }; + } + return CircuitView::Open { remaining: Duration::ZERO }; + } + CircuitState::HalfOpenInFlight => { + return CircuitView::HalfOpenInFlight; + } + CircuitState::Closed => {} + } + } + + let Some(entry) = self.shared.circuits.get(key) else { + return CircuitView::Closed; + }; + match entry.state { + CircuitState::Closed => { + if entry.failures > 0 { + CircuitView::Counting(entry.failures) + } else { + CircuitView::Closed + } + } + CircuitState::OpenUntil(until) => { + if now < until { + CircuitView::Open { remaining: until.saturating_duration_since(now) } + } else { + CircuitView::Open { remaining: Duration::ZERO } + } + } + CircuitState::HalfOpenInFlight => CircuitView::HalfOpenInFlight, + } + } + + pub fn input_circuit_view(&self, input_name: &str, now: Instant) -> CircuitView { + let Some(entry) = self.shared.input_circuits.get(input_name) else { + return CircuitView::Closed; + }; + match entry.circuit.state { + CircuitState::Closed => { + let policy = self.policy.load(); + let window = policy.input_window; + let mut distinct = std::collections::HashSet::new(); + let mut count = 0usize; + for (k, ts) in &entry.recent { + let in_window = if *ts <= now { now.duration_since(*ts) <= window } else { true }; + if in_window { + count += 1; + distinct.insert(&k.1); + } + } + if distinct.len() >= 2 && count > 0 { + CircuitView::Counting(u8::try_from(count).unwrap_or(u8::MAX)) + } else { + CircuitView::Closed + } + } + CircuitState::OpenUntil(until) => { + if now < until { + CircuitView::Open { remaining: until.saturating_duration_since(now) } + } else { + CircuitView::Open { remaining: Duration::ZERO } + } + } + CircuitState::HalfOpenInFlight => CircuitView::HalfOpenInFlight, + } + } + + pub fn set_policy(&self, policy: CircuitPolicy) { self.policy.store(Arc::new(policy)); } +} + +#[cfg(test)] +mod tests { + use super::*; + use shared::model::{PlaylistItemType, VirtualId, XtreamCluster}; + + fn qos(score_24h: u8, score_7d: u8, confidence: u8) -> QosPart { + QosPart { score_24h, score_7d, confidence, failure_streak: 0 } + } + + fn test_member(input: &str, stream_id: &str, provider_id: u32, member_vid: u32) -> GroupMember { + GroupMember { + member_vid: VirtualId::new(member_vid), + input_name: input.into(), + provider_id, + input_stream_id: stream_id.into(), + url: format!("http://{input}.invalid/{stream_id}.ts").into(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + title: "ZDF".into(), + archive: None, + epg_channel_id: None, + additional_properties: None, + } + } + + #[test] + fn qos_lookup_uses_provider_id_or_target_member() { + let index = MemberHealthIndex::new(CircuitPolicy::default()); + let target: Arc = "Main".into(); + let xtream = test_member("solis", "1", 1, 101); + let m3u = test_member("m3u-in", "chan-a", 0, 202); + index.replace_qos([ + (QosKey::Provider { input: "solis".into(), provider_id: 1 }, qos(80, 70, 20)), + (QosKey::Provider { input: "solis".into(), provider_id: 1 }, qos(40, 50, 90)), + (QosKey::TargetMember { target: Arc::clone(&target), member_vid: 202 }, qos(66, 66, 50)), + ]); + let now = Instant::now(); + assert_eq!(index.health(&target, &xtream, now).qos.map(|q| q.score_24h), Some(40)); + assert_eq!(index.health(&target, &m3u, now).qos.map(|q| q.score_24h), Some(66)); + } + + #[test] + fn probe_patch_keeps_qos() { + let index = MemberHealthIndex::new(CircuitPolicy::default()); + let target: Arc = "Main".into(); + let member = test_member("solis", "1", 1, 101); + index.replace_qos([(QosKey::Provider { input: "solis".into(), provider_id: 1 }, qos(40, 50, 90))]); + index.patch_probe(&member_key(&member), Some(r#"{"codec_name":"hevc","width":3840,"height":2160}"#), 12_000); + let health = index.health(&target, &member, Instant::now()); + assert_eq!(health.qos.map(|q| q.score_24h), Some(40)); + let probe = health.probe.expect("probe"); + assert_eq!(probe.resolution, VideoResolution::P2160); + assert_eq!(probe.codec, VideoCodec::H265); + assert_eq!(probe.bitrate_kbps, 12_000); + } + + #[test] + fn audio_patch_sets_codec_channels_and_language() { + let index = MemberHealthIndex::new(CircuitPolicy::default()); + let target: Arc = "Main".into(); + let member = test_member("solis", "1", 1, 101); + index.patch_probe(&member_key(&member), Some(r#"{"codec_name":"h264","width":1920,"height":1080}"#), 5_000); + index.patch_probe_audio( + &member_key(&member), + Some(r#"[{"codec_name":"ac3","channels":6,"tags":{"language":"DEU"}}]"#), + ); + let probe = index.health(&target, &member, Instant::now()).probe.expect("probe"); + assert_eq!(probe.codec, VideoCodec::H264); + assert_eq!(probe.audio_codec, AudioCodec::AC3); + assert_eq!(probe.audio_channels, AudioChannels::Surround51); + assert_eq!(probe.audio_language.as_deref(), Some("deu")); + } +} diff --git a/backend/session/src/channel_group/mod.rs b/backend/session/src/channel_group/mod.rs new file mode 100644 index 000000000..805666428 --- /dev/null +++ b/backend/session/src/channel_group/mod.rs @@ -0,0 +1,9 @@ +pub mod circuit; +pub mod health; +pub mod ranking; +#[cfg(test)] +pub(crate) mod test_support; + +pub use circuit::*; +pub use health::*; +pub use ranking::*; diff --git a/backend/session/src/channel_group/ranking.rs b/backend/session/src/channel_group/ranking.rs new file mode 100644 index 000000000..a86ba15a1 --- /dev/null +++ b/backend/session/src/channel_group/ranking.rs @@ -0,0 +1,441 @@ +use crate::channel_group::{ + circuit::{CircuitView, MemberKey}, + health::{member_key, MemberHealthIndex}, +}; +use shared::{ + model::{AudioChannels, AudioCodec, ChannelGroupPolicy, RuntimeAudioLanguage, VideoCodec, VideoResolution}, + utils::quality_rank, +}; +use std::{cmp::Reverse, sync::Arc, time::Instant}; +use tuliprox_repository::GroupMember; + +pub const W_QOS: i32 = 6; +pub const W_QUALITY: i32 = 3; +pub const W_ORDER: i32 = 1; +pub const NEUTRAL_QOS: u8 = 50; +pub const OPEN_CIRCUIT_PENALTY: i32 = 100; +pub const SAME_CODEC_BONUS: i32 = 10; +pub const SAME_RESOLUTION_BONUS: i32 = 10; +pub const SAME_AUDIO_BONUS: i32 = 5; +pub const SAME_LANGUAGE_BONUS: i32 = 5; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioSignature { + pub codec: AudioCodec, + pub channels: AudioChannels, + pub language: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SelectionMode { + Live, + Runtime { + current_codec: VideoCodec, + current_resolution: VideoResolution, + current_audio: Option, + audio_language: RuntimeAudioLanguage, + }, + Archive { + start_ts: i64, + now_ts: i64, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +pub struct ScoreBreakdown { + pub qos: u8, + pub quality: u8, + pub order: u8, + pub total: u8, + pub demoted: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RankedCandidate { + pub index: usize, + pub key: MemberKey, + pub score: ScoreBreakdown, +} + +fn filter_candidates<'a>( + target_name: &Arc, + members: &'a [GroupMember], + health: &MemberHealthIndex, + mode: &SelectionMode, + now: Instant, +) -> Vec<(usize, &'a GroupMember)> { + let mut candidates = Vec::with_capacity(members.len()); + for (i, member) in members.iter().enumerate() { + match mode { + SelectionMode::Live => { + candidates.push((i, member)); + } + SelectionMode::Archive { start_ts, now_ts } => { + if let Some(archive) = member.archive { + let earliest_ts = now_ts.saturating_sub(i64::from(archive.duration_days) * 86_400); + if *start_ts >= earliest_ts { + candidates.push((i, member)); + } + } + } + SelectionMode::Runtime { current_audio, audio_language, .. } => { + if *audio_language == RuntimeAudioLanguage::Require { + if let Some(target_lang) = current_audio.as_ref().and_then(|a| a.language.as_deref()) { + let m_health = health.health(target_name, member, now); + if let Some(cand_lang) = m_health.probe.as_ref().and_then(|p| p.audio_language.as_deref()) { + if !cand_lang.eq_ignore_ascii_case(target_lang) { + continue; + } + } + } + } + candidates.push((i, member)); + } + } + } + candidates +} + +fn compute_quality_points( + probe: Option<&crate::channel_group::health::ProbePart>, + title: &str, + min_bitrate: u32, + max_bitrate: u32, +) -> u8 { + let (res_pts, bitrate_pts) = if let Some(probe) = probe { + let res = match probe.resolution { + VideoResolution::SD => 25, + VideoResolution::P720 => 50, + VideoResolution::P1080 => 75, + VideoResolution::P1440 | VideoResolution::P2160 | VideoResolution::P4320 => 100, + VideoResolution::Unknown => match quality_rank(title) { + 1 => 25, + 2 => 50, + 3 => 75, + 4 | 5 => 100, + _ => 0, + }, + }; + let b_pts = if max_bitrate > min_bitrate { + u32::try_from( + 10 * u64::from(probe.bitrate_kbps.saturating_sub(min_bitrate)) / u64::from(max_bitrate - min_bitrate), + ) + .unwrap_or(0) + } else { + 0 + }; + (res, b_pts) + } else { + let res = match quality_rank(title) { + 1 => 25, + 2 => 50, + 3 => 75, + 4 | 5 => 100, + _ => 0, + }; + (res, 0) + }; + u8::try_from((res_pts + bitrate_pts).min(100)).unwrap_or(100) +} + +fn compute_runtime_bonuses(probe: Option<&crate::channel_group::health::ProbePart>, mode: &SelectionMode) -> i32 { + let SelectionMode::Runtime { current_codec, current_resolution, current_audio, .. } = mode else { + return 0; + }; + let Some(probe) = probe else { + return 0; + }; + + let mut bonuses = 0i32; + if *current_codec != VideoCodec::Other && probe.codec == *current_codec { + bonuses += SAME_CODEC_BONUS; + } + if *current_resolution != VideoResolution::Unknown && probe.resolution == *current_resolution { + bonuses += SAME_RESOLUTION_BONUS; + } + if let Some(curr_a) = current_audio { + if curr_a.codec != AudioCodec::Other + && probe.audio_codec == curr_a.codec + && probe.audio_channels == curr_a.channels + { + bonuses += SAME_AUDIO_BONUS; + } + if let Some(curr_lang) = &curr_a.language { + if let Some(cand_lang) = &probe.audio_language { + if cand_lang.eq_ignore_ascii_case(curr_lang) { + bonuses += SAME_LANGUAGE_BONUS; + } + } + } + } + bonuses +} + +pub fn rank_members( + target_name: &Arc, + members: &[GroupMember], + policy: ChannelGroupPolicy, + health: &MemberHealthIndex, + mode: &SelectionMode, + now: Instant, +) -> Vec { + if members.is_empty() { + return Vec::new(); + } + + let candidates = filter_candidates(target_name, members, health, mode, now); + if candidates.is_empty() { + return Vec::new(); + } + + let member_healths: Vec<_> = candidates.iter().map(|(_, m)| health.health(target_name, m, now)).collect(); + + let mut max_bitrate = 0u32; + let mut min_bitrate = u32::MAX; + for h in &member_healths { + let b = h.probe.as_ref().map_or(0, |p| p.bitrate_kbps); + max_bitrate = max_bitrate.max(b); + min_bitrate = min_bitrate.min(b); + } + if min_bitrate == u32::MAX { + min_bitrate = 0; + } + + let mut ranked = Vec::with_capacity(candidates.len()); + for ((index, member), m_health) in candidates.into_iter().zip(member_healths) { + let key = member_key(member); + + let member_circuit_open = matches!(m_health.circuit, CircuitView::Open { .. }); + let input_circuit_open = matches!(health.input_circuit_view(&member.input_name, now), CircuitView::Open { .. }); + let high_failure_streak = m_health.qos.is_some_and(|q| q.failure_streak >= 3); + let demoted = member_circuit_open || input_circuit_open || high_failure_streak; + + let qos = m_health.qos.map_or(NEUTRAL_QOS, |q| { + let conf = u32::from(q.confidence); + let s24 = u32::from(q.score_24h); + let s7d = u32::from(q.score_7d); + u8::try_from((s24 * conf + s7d * (100 - conf)) / 100).unwrap_or(0) + }); + + let quality = compute_quality_points(m_health.probe.as_ref(), member.title.as_ref(), min_bitrate, max_bitrate); + let order = u8::try_from(100 - (100 * index / members.len())).unwrap_or(0); + let base_total = (W_QOS * i32::from(qos) + W_QUALITY * i32::from(quality) + W_ORDER * i32::from(order)) / 10; + let bonuses = compute_runtime_bonuses(m_health.probe.as_ref(), mode); + let penalty = if member_circuit_open || input_circuit_open { OPEN_CIRCUIT_PENALTY } else { 0 }; + let total = u8::try_from((base_total + bonuses - penalty).clamp(0, 100)).unwrap_or(0); + + ranked.push(RankedCandidate { index, key, score: ScoreBreakdown { qos, quality, order, total, demoted } }); + } + + match policy { + ChannelGroupPolicy::Auto => { + ranked.sort_by_key(|c| (c.score.demoted, Reverse(c.score.total), c.index)); + } + ChannelGroupPolicy::Priority => { + ranked.sort_by_key(|c| (c.score.demoted, c.index)); + } + } + + ranked +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::channel_group::{ + circuit::{CircuitPolicy, FailureClass}, + health::{member_key, QosKey, QosPart}, + test_support::test_member, + }; + use std::time::Duration; + use tuliprox_repository::MemberArchive; + + const TARGET: &str = "Main"; + + fn fixture(spec: &[(&str, u32, Option, &str)]) -> (Vec, MemberHealthIndex) { + let index = MemberHealthIndex::new(CircuitPolicy::new(1, Duration::from_secs(60), Duration::from_secs(60))); + let mut members = Vec::new(); + for (i, (input, provider_id, qos_24h, probe)) in spec.iter().enumerate() { + let member = test_member(input, &provider_id.to_string(), *provider_id, 100 + u32::try_from(i).unwrap()); + if let Some(score) = qos_24h { + index.replace_qos_merge([( + QosKey::Provider { input: (*input).into(), provider_id: *provider_id }, + QosPart { score_24h: *score, score_7d: *score, confidence: 100, failure_streak: 0 }, + )]); + } + if !probe.is_empty() { + let (codec, dims) = probe.split_once(':').unwrap_or(("h264", probe)); + let (w, h) = dims.split_once('x').expect("WxH"); + let json = format!(r#"{{"codec_name":"{codec}","width":{w},"height":{h}}}"#); + index.patch_probe(&member_key(&member), Some(&json), 5_000); + } + members.push(member); + } + (members, index) + } + + fn fixture_names(titles: &[&str]) -> (Vec, MemberHealthIndex) { + let (mut members, index) = fixture( + &titles + .iter() + .enumerate() + .map(|(i, _)| ("solis", u32::try_from(i + 1).unwrap(), None, "")) + .collect::>(), + ); + for (member, title) in members.iter_mut().zip(titles) { + member.title = (*title).into(); + } + (members, index) + } + + fn open_circuit(index: &MemberHealthIndex, member: &GroupMember) { + index.record_failure(&member_key(member), Instant::now()); + } + + fn rank( + members: &[GroupMember], + policy: ChannelGroupPolicy, + index: &MemberHealthIndex, + mode: &SelectionMode, + ) -> Vec { + rank_members(&Arc::from(TARGET), members, policy, index, mode, Instant::now()) + } + + #[test] + fn auto_prefers_uhd_over_one_point_better_qos() { + let (members, health) = fixture(&[("solis", 1, Some(81), "1920x1080"), ("korath", 7, Some(80), "3840x2160")]); + assert_eq!(rank(&members, ChannelGroupPolicy::Auto, &health, &SelectionMode::Live)[0].index, 1); + } + + #[test] + fn auto_prefers_much_better_qos_over_resolution() { + let (members, health) = fixture(&[("solis", 1, Some(95), "1920x1080"), ("korath", 7, Some(30), "3840x2160")]); + assert_eq!(rank(&members, ChannelGroupPolicy::Auto, &health, &SelectionMode::Live)[0].index, 0); + } + + #[test] + fn score_breakdown_uses_fixed_weights() { + let (members, health) = fixture(&[("solis", 1, Some(100), "3840x2160")]); + let ranked = rank(&members, ChannelGroupPolicy::Auto, &health, &SelectionMode::Live); + assert_eq!( + (ranked[0].score.qos, ranked[0].score.quality, ranked[0].score.order, ranked[0].score.total), + (100, 100, 100, 100) + ); + } + + #[test] + fn missing_data_is_neutral_and_uses_name_quality() { + let (members, health) = fixture_names(&["ZDF SD", "ZDF FHD"]); + let ranked = rank(&members, ChannelGroupPolicy::Auto, &health, &SelectionMode::Live); + assert_eq!(ranked[0].index, 1); + assert_eq!(ranked[0].score.qos, 50); + } + + #[test] + fn priority_keeps_order_but_demotes_open_circuit_to_end() { + let (members, health) = + fixture(&[("solis", 1, Some(10), "720x576"), ("korath", 7, Some(99), "3840x2160"), ("veyra", 3, None, "")]); + open_circuit(&health, &members[0]); + let ranked = rank(&members, ChannelGroupPolicy::Priority, &health, &SelectionMode::Live); + assert_eq!(ranked.iter().map(|c| c.index).collect::>(), vec![1, 2, 0]); + assert!(ranked[2].score.demoted); + } + + #[test] + fn open_input_circuit_demotes_all_members_of_that_input() { + let (members, health) = + fixture(&[("solis", 1, Some(99), ""), ("solis", 2, Some(99), ""), ("korath", 7, Some(40), "")]); + health.set_policy( + CircuitPolicy::new(5, Duration::from_secs(60), Duration::from_secs(60)) + .with_input(2, Duration::from_secs(120)), + ); + let now = Instant::now(); + health.record_classified_failure(&member_key(&members[0]), FailureClass::Transport, now); + health.record_classified_failure(&member_key(&members[1]), FailureClass::Transport, now); + let ranked = rank(&members, ChannelGroupPolicy::Auto, &health, &SelectionMode::Live); + assert_eq!(ranked[0].index, 2); + assert!(ranked[1].score.demoted && ranked[2].score.demoted); + } + + #[test] + fn open_circuit_penalty_saturates_at_zero() { + let (members, health) = fixture(&[("solis", 1, Some(0), "720x576")]); + open_circuit(&health, &members[0]); + assert_eq!(rank(&members, ChannelGroupPolicy::Auto, &health, &SelectionMode::Live)[0].score.total, 0); + } + + #[test] + fn runtime_mode_prefers_same_codec() { + let (members, health) = + fixture(&[("solis", 1, Some(70), "hevc:3840x2160"), ("korath", 7, Some(70), "h264:1920x1080")]); + let mode = SelectionMode::Runtime { + current_codec: VideoCodec::H264, + current_resolution: VideoResolution::P1080, + current_audio: None, + audio_language: RuntimeAudioLanguage::Prefer, + }; + assert_eq!(rank(&members, ChannelGroupPolicy::Auto, &health, &mode)[0].index, 1); + } + + fn set_audio(health: &MemberHealthIndex, member: &GroupMember, json: &str) { + health.patch_probe_audio(&member_key(member), Some(json)); + } + + fn runtime_with_audio(language: Option<&str>, policy: RuntimeAudioLanguage) -> SelectionMode { + SelectionMode::Runtime { + current_codec: VideoCodec::H264, + current_resolution: VideoResolution::P1080, + current_audio: Some(AudioSignature { + codec: AudioCodec::AC3, + channels: AudioChannels::Surround51, + language: language.map(Into::into), + }), + audio_language: policy, + } + } + + #[test] + fn runtime_mode_prefers_same_audio_and_language() { + let (members, health) = + fixture(&[("solis", 1, Some(70), "h264:1920x1080"), ("korath", 7, Some(70), "h264:1920x1080")]); + set_audio(&health, &members[0], r#"[{"codec_name":"aac","channels":2,"tags":{"language":"eng"}}]"#); + set_audio(&health, &members[1], r#"[{"codec_name":"ac3","channels":6,"tags":{"language":"deu"}}]"#); + let ranked = rank( + &members, + ChannelGroupPolicy::Auto, + &health, + &runtime_with_audio(Some("deu"), RuntimeAudioLanguage::Prefer), + ); + assert_eq!(ranked[0].index, 1); + assert_eq!(ranked.len(), 2); + } + + #[test] + fn require_language_drops_known_mismatch_but_keeps_unknown() { + let (members, health) = + fixture(&[("solis", 1, Some(90), ""), ("korath", 7, Some(50), ""), ("veyra", 3, Some(40), "")]); + set_audio(&health, &members[0], r#"[{"codec_name":"aac","channels":2,"tags":{"language":"eng"}}]"#); + set_audio(&health, &members[1], r#"[{"codec_name":"aac","channels":2}]"#); // unknown language + set_audio(&health, &members[2], r#"[{"codec_name":"aac","channels":2,"tags":{"language":"deu"}}]"#); + let ranked = rank( + &members, + ChannelGroupPolicy::Auto, + &health, + &runtime_with_audio(Some("deu"), RuntimeAudioLanguage::Require), + ); + assert_eq!(ranked.iter().map(|c| c.index).collect::>(), vec![1, 2]); + } + + #[test] + fn archive_mode_filters_members_not_covering_start() { + let (mut members, health) = fixture(&[("solis", 1, Some(90), ""), ("korath", 7, Some(50), "")]); + members[0].archive = Some(MemberArchive { duration_days: 1 }); + members[1].archive = Some(MemberArchive { duration_days: 7 }); + let now_ts = 1_700_000_000; + let mode = SelectionMode::Archive { start_ts: now_ts - 3 * 86_400, now_ts }; + assert_eq!( + rank(&members, ChannelGroupPolicy::Auto, &health, &mode).iter().map(|c| c.index).collect::>(), + vec![1] + ); + } +} diff --git a/backend/session/src/channel_group/test_support.rs b/backend/session/src/channel_group/test_support.rs new file mode 100644 index 000000000..6ebe1ad1d --- /dev/null +++ b/backend/session/src/channel_group/test_support.rs @@ -0,0 +1,18 @@ +use shared::model::{PlaylistItemType, VirtualId, XtreamCluster}; +use tuliprox_repository::GroupMember; + +pub fn test_member(input: &str, stream_id: &str, provider_id: u32, member_vid: u32) -> GroupMember { + GroupMember { + member_vid: VirtualId::new(member_vid), + input_name: input.into(), + provider_id, + input_stream_id: stream_id.into(), + url: format!("http://{input}.invalid/{stream_id}.ts").into(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + title: "ZDF".into(), + archive: None, + epg_channel_id: None, + additional_properties: None, + } +} diff --git a/backend/session/src/connection_manager/tests.rs b/backend/session/src/connection_manager/tests.rs index dc8e97da3..937a01a27 100644 --- a/backend/session/src/connection_manager/tests.rs +++ b/backend/session/src/connection_manager/tests.rs @@ -48,6 +48,7 @@ fn make_stream_info(provider: &str, title: &str) -> StreamInfo { epg_channel_id: None, epg_reference_ts: None, upstream_user_agent: None, + group_virtual_id: None, }; StreamInfo::new(shared::model::StreamInfoParams { uid: 0, diff --git a/backend/session/src/event_manager.rs b/backend/session/src/event_manager.rs index d07b45da9..399add313 100644 --- a/backend/session/src/event_manager.rs +++ b/backend/session/src/event_manager.rs @@ -157,6 +157,7 @@ struct LatchedEvents { pub struct EventManager { pub(crate) account_provider: std::sync::OnceLock>, + pub(crate) member_health: std::sync::OnceLock>, channel_tx: tokio::sync::broadcast::Sender, meters: StreamMeterRegistry, stats: Arc, @@ -199,6 +200,7 @@ impl EventManager { Self { channel_tx, account_provider: std::sync::OnceLock::new(), + member_health: std::sync::OnceLock::new(), meters: StreamMeterRegistry::new(), stats: Arc::new(EventBusStats::default()), last_nudge: Mutex::new(HashMap::new()), @@ -391,6 +393,10 @@ impl FilteredEventReceiver { } impl EventManager { + pub fn bind_member_health(&self, health: &Arc) { + let _ = self.member_health.set(Arc::downgrade(health)); + } + /// Stop background work, flushing what is pending. /// /// The event channel needs nothing: closing the sender is what tells @@ -417,6 +423,24 @@ impl EventSink for EventManager { }); } } + fn observe_stream_probe( + &self, + input_name: Arc, + input_stream_id: Arc, + video: Option<&str>, + audio: Option<&str>, + bitrate_kbps: u32, + ) { + if let Some(health) = self.member_health.get().and_then(std::sync::Weak::upgrade) { + let key = (input_name, input_stream_id); + if video.is_some() || bitrate_kbps > 0 { + health.patch_probe(&key, video, bitrate_kbps); + } + if audio.is_some() { + health.patch_probe_audio(&key, audio); + } + } + } } impl Default for EventManager { diff --git a/backend/session/src/lib.rs b/backend/session/src/lib.rs index 35031296e..2b8c262ae 100644 --- a/backend/session/src/lib.rs +++ b/backend/session/src/lib.rs @@ -21,6 +21,7 @@ pub mod active_provider_manager; pub mod active_user_manager; pub mod admission; pub mod admission_strategy; +pub mod channel_group; pub mod connection_manager; pub mod event_manager; pub mod meter; @@ -37,7 +38,7 @@ pub mod stream_options; pub mod streams; pub use self::{ - active_provider_manager::*, active_user_manager::*, admission::*, admission_strategy::*, connection_manager::*, - event_manager::*, meter::*, meter_registry::*, provider_leases::*, provider_lineup_manager::*, - provider_session_headers::*, stream::*, streams::*, + active_provider_manager::*, active_user_manager::*, admission::*, admission_strategy::*, channel_group::*, + connection_manager::*, event_manager::*, meter::*, meter_registry::*, provider_leases::*, + provider_lineup_manager::*, provider_session_headers::*, stream::*, streams::*, }; diff --git a/backend/session/src/provider_lineup_manager.rs b/backend/session/src/provider_lineup_manager.rs index 8112adb59..38c72f51c 100644 --- a/backend/session/src/provider_lineup_manager.rs +++ b/backend/session/src/provider_lineup_manager.rs @@ -156,6 +156,26 @@ impl ProviderLineup { } } } + + fn probe_excluding( + &self, + with_grace: bool, + grace_period_timeout_secs: u64, + excluded_providers: &std::collections::HashSet>, + ) -> ProviderAllocation { + match self { + ProviderLineup::Single(lineup) => { + if excluded_providers.contains(&lineup.provider.name) { + ProviderAllocation::Exhausted + } else { + lineup.probe(with_grace, grace_period_timeout_secs) + } + } + ProviderLineup::Multi(lineup) => { + lineup.probe_excluding(with_grace, grace_period_timeout_secs, excluded_providers) + } + } + } } /// Handles a single provider and ensures safe allocation/release of connections. @@ -183,6 +203,10 @@ impl SingleProviderLineup { self.provider.try_allocate(with_grace, grace_period_timeout_secs) } + fn probe(&self, with_grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { + self.provider.probe_capacity(with_grace, grace_period_timeout_secs) + } + #[cfg(test)] fn release(&self, provider_name: &Arc) { if &self.provider.name == provider_name { @@ -337,6 +361,63 @@ impl MultiProviderLineup { ProviderAllocation::Exhausted } + fn probe_next_provider_from_group( + priority_group: &ProviderPriorityGroup, + grace: bool, + grace_period_timeout_secs: u64, + excluded_providers: &std::collections::HashSet>, + ) -> ProviderAllocation { + match priority_group { + ProviderPriorityGroup::SingleProviderGroup(p) => { + if excluded_providers.contains(&p.name) { + return ProviderAllocation::Exhausted; + } + let result = p.probe_capacity(grace, grace_period_timeout_secs); + if !matches!(result, ProviderAllocation::Exhausted) { + return result; + } + } + ProviderPriorityGroup::MultiProviderGroup(index, pg) => { + let provider_count = pg.len(); + if provider_count == 0 { + return ProviderAllocation::Exhausted; + } + if grace { + for p in pg { + if excluded_providers.contains(&p.name) { + continue; + } + let result = p.probe_capacity(true, grace_period_timeout_secs); + if !matches!(result, ProviderAllocation::Exhausted) { + return result; + } + } + return ProviderAllocation::Exhausted; + } + + let start = index.load(Ordering::Acquire) % provider_count; + let mut idx = start; + + loop { + let p = &pg[idx]; + if !excluded_providers.contains(&p.name) { + let result = p.probe_capacity(grace, grace_period_timeout_secs); + if !matches!(result, ProviderAllocation::Exhausted) { + return result; + } + } + + idx = (idx + 1) % provider_count; + + if idx == start { + break; + } + } + } + } + ProviderAllocation::Exhausted + } + // Used for redirect to cyclce through provider fn get_next_provider_from_group( priority_group: &ProviderPriorityGroup, @@ -425,6 +506,24 @@ impl MultiProviderLineup { .unwrap_or(ProviderAllocation::Exhausted) } + fn probe_excluding( + &self, + with_grace: bool, + grace_period_timeout_secs: u64, + excluded_providers: &std::collections::HashSet>, + ) -> ProviderAllocation { + self.scan_priority_groups(with_grace, |priority_group, grace| { + let allocation = Self::probe_next_provider_from_group( + priority_group, + grace, + grace_period_timeout_secs, + excluded_providers, + ); + (!matches!(allocation, ProviderAllocation::Exhausted)).then_some(allocation) + }) + .unwrap_or(ProviderAllocation::Exhausted) + } + // It is intended to use with redirects to cycle through providers. fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { self.scan_priority_groups(true, |priority_group, grace| { @@ -930,6 +1029,40 @@ impl ProviderLineupManager { allocation } + pub fn probe_exact_connection_with_grace_override( + &self, + provider_name: &Arc, + allow_grace: bool, + ) -> ProviderAllocation { + let snapshot = self.snapshot.load_full(); + let with_grace = allow_grace && self.grace_period_millis.load(Ordering::Acquire) > 0; + match Self::get_provider_config_by_name(provider_name, &snapshot.providers) { + None => ProviderAllocation::Exhausted, + Some((_lineup, config)) => { + config.probe_capacity(with_grace, self.grace_period_timeout_secs.load(Ordering::Acquire)) + } + } + } + + pub fn probe_connection_with_grace_override_excluding( + &self, + input_name: &Arc, + allow_grace: bool, + excluded_providers: &std::collections::HashSet>, + ) -> ProviderAllocation { + let snapshot = self.snapshot.load_full(); + let lineup_opt = Self::get_provider_config_by_name(input_name, &snapshot.providers); + let with_grace = allow_grace && self.grace_period_millis.load(Ordering::Acquire) > 0; + match lineup_opt { + None => ProviderAllocation::Exhausted, + Some((lineup, _config)) => lineup.probe_excluding( + with_grace, + self.grace_period_timeout_secs.load(Ordering::Acquire), + excluded_providers, + ), + } + } + /// Report that every provider behind an input is at capacity. /// /// The snapshot used to be built only when debug logging was on and diff --git a/backend/session/src/qos_aggregation_manager.rs b/backend/session/src/qos_aggregation_manager.rs index 0a7d19034..5ed6f2bbf 100644 --- a/backend/session/src/qos_aggregation_manager.rs +++ b/backend/session/src/qos_aggregation_manager.rs @@ -47,7 +47,19 @@ pub fn qos_aggregation_is_enabled(config: &Config) -> bool { }) } +pub trait QosAggregationListener: Send + Sync { + fn on_snapshots(&self, snapshots: &[QosSnapshotRecord]); +} + pub fn exec_qos_aggregation(app_config: &Arc, cancel_token: &CancellationToken) { + exec_qos_aggregation_with_listener(app_config, cancel_token, None); +} + +pub fn exec_qos_aggregation_with_listener( + app_config: &Arc, + cancel_token: &CancellationToken, + listener: Option>, +) { let config = app_config.config.load(); let Some(reverse_proxy) = config.reverse_proxy.as_ref() else { return; @@ -81,8 +93,15 @@ pub fn exec_qos_aggregation(app_config: &Arc, cancel_token: &Cancella let today = current_utc_day(); let repo_clone = repo.clone(); let history_dir_clone = history_dir.clone(); + let listener_clone = listener.clone(); if let Err(err) = tokio::task::spawn_blocking(move || { - run_aggregation_once(&repo_clone, &history_dir_clone, &today, compaction_interval_secs) + run_aggregation_once(&repo_clone, &history_dir_clone, &today, compaction_interval_secs)?; + if let Some(listener) = listener_clone { + let mut snapshots = Vec::new(); + let _ = repo_clone.for_each_snapshot(|s| snapshots.push(s.clone())); + listener.on_snapshots(&snapshots); + } + Ok::<(), std::io::Error>(()) }) .await .unwrap_or_else(|e| Err(std::io::Error::other(e.to_string()))) diff --git a/backend/session/src/streams/shared_stream_manager/tests.rs b/backend/session/src/streams/shared_stream_manager/tests.rs index 97e5e49e1..8aa799cb1 100644 --- a/backend/session/src/streams/shared_stream_manager/tests.rs +++ b/backend/session/src/streams/shared_stream_manager/tests.rs @@ -91,6 +91,7 @@ fn create_test_stream_channel(url: &str) -> StreamChannel { epg_channel_id: None, epg_reference_ts: None, upstream_user_agent: None, + group_virtual_id: None, } } diff --git a/frontend/src/app/components/dashboard/stream_display/helpers.rs b/frontend/src/app/components/dashboard/stream_display/helpers.rs index feae9d12b..7439ed8f6 100644 --- a/frontend/src/app/components/dashboard/stream_display/helpers.rs +++ b/frontend/src/app/components/dashboard/stream_display/helpers.rs @@ -250,6 +250,7 @@ mod tests { channel: StreamChannel { target_id: 1, virtual_id: uid, + group_virtual_id: None, provider_id: 1, item_type, cluster: item_type.cluster(), diff --git a/frontend/src/app/components/dashboard/stream_display/mod.rs b/frontend/src/app/components/dashboard/stream_display/mod.rs index 02e232e07..7ae097543 100644 --- a/frontend/src/app/components/dashboard/stream_display/mod.rs +++ b/frontend/src/app/components/dashboard/stream_display/mod.rs @@ -352,6 +352,7 @@ mod tests { channel: StreamChannel { target_id: 1, virtual_id: 42, + group_virtual_id: None, provider_id: 1, item_type, cluster: XtreamCluster::Live, diff --git a/frontend/src/hooks/use_server_status.rs b/frontend/src/hooks/use_server_status.rs index 124074f8f..3f2717651 100644 --- a/frontend/src/hooks/use_server_status.rs +++ b/frontend/src/hooks/use_server_status.rs @@ -154,6 +154,7 @@ fn download_task_to_stream_with_ts(download: &RecordingTaskDto, ts: u64) -> Stre channel: StreamChannel { target_id: 0, virtual_id: uid, + group_virtual_id: None, provider_id: 0, input_name: "".intern(), item_type, @@ -466,6 +467,7 @@ mod tests { channel: StreamChannel { target_id: 1, virtual_id: 1, + group_virtual_id: None, provider_id: 1, item_type, cluster: XtreamCluster::Live, diff --git a/shared/src/error/tuliprox_error.rs b/shared/src/error/tuliprox_error.rs index cf15225f3..4682c7eb8 100644 --- a/shared/src/error/tuliprox_error.rs +++ b/shared/src/error/tuliprox_error.rs @@ -110,6 +110,7 @@ error_kinds! { ConfigVideoDownload => "config video download error", ConfigTarget => "config target error", TargetBouquet => "target bouquet error", + ChannelGroup => "channel group error", ConfigWebUi => "config web UI error", RepositoryEpg => "repository EPG error", RepositoryXtream => "repository XTream error", diff --git a/shared/src/model/channel_group.rs b/shared/src/model/channel_group.rs new file mode 100644 index 000000000..ca201545d --- /dev/null +++ b/shared/src/model/channel_group.rs @@ -0,0 +1,152 @@ +use crate::model::ChannelGroupPolicy; +use std::collections::HashSet; + +pub const CHANNEL_GROUP_OVERRIDES_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] +pub enum CapacityTier { + Free, + OwnRelease, + Grace, + Preempt, + Exhausted, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CircuitState { + Closed, + OpenUntil(std::time::Instant), + HalfOpenInFlight, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] +pub struct ChannelMemberRefDto { + pub input: String, + pub stream_id: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ManualChannelGroupDto { + pub id: String, + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logo: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epg_channel_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub playlist_group: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub policy: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub members: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct AutoGroupOverrideDto { + pub auto_key: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub policy: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub order: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub add: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub remove: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logo: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epg_channel_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub playlist_group: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ChannelGroupOverridesDto { + pub version: u32, + pub target: String, + #[serde(default)] + pub revision: u64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub groups: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub auto_overrides: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub excluded: Vec, +} + +impl ChannelGroupOverridesDto { + pub fn new(target: &str) -> Self { + Self { + version: CHANNEL_GROUP_OVERRIDES_VERSION, + target: target.to_string(), + revision: 0, + groups: Vec::new(), + auto_overrides: Vec::new(), + excluded: Vec::new(), + } + } + + pub fn canonicalize(&mut self) { + self.target = self.target.trim().to_string(); + for group in &mut self.groups { + group.id = group.id.trim().to_string(); + group.name = group.name.trim().to_string(); + group.logo = group.logo.as_ref().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + group.epg_channel_id = + group.epg_channel_id.as_ref().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + group.playlist_group = + group.playlist_group.as_ref().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + for m in &mut group.members { + m.input = m.input.trim().to_string(); + m.stream_id = m.stream_id.trim().to_string(); + } + let mut seen = HashSet::new(); + group.members.retain(|m| { + !m.input.is_empty() && !m.stream_id.is_empty() && seen.insert((m.input.clone(), m.stream_id.clone())) + }); + } + for auto in &mut self.auto_overrides { + auto.auto_key = auto.auto_key.trim().to_string(); + auto.name = auto.name.as_ref().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + auto.logo = auto.logo.as_ref().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + auto.epg_channel_id = auto.epg_channel_id.as_ref().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + auto.playlist_group = auto.playlist_group.as_ref().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + for m in &mut auto.order { + m.input = m.input.trim().to_string(); + m.stream_id = m.stream_id.trim().to_string(); + } + let mut seen = HashSet::new(); + auto.order.retain(|m| { + !m.input.is_empty() && !m.stream_id.is_empty() && seen.insert((m.input.clone(), m.stream_id.clone())) + }); + + for m in &mut auto.add { + m.input = m.input.trim().to_string(); + m.stream_id = m.stream_id.trim().to_string(); + } + seen.clear(); + auto.add.retain(|m| { + !m.input.is_empty() && !m.stream_id.is_empty() && seen.insert((m.input.clone(), m.stream_id.clone())) + }); + + for m in &mut auto.remove { + m.input = m.input.trim().to_string(); + m.stream_id = m.stream_id.trim().to_string(); + } + seen.clear(); + auto.remove.retain(|m| { + !m.input.is_empty() && !m.stream_id.is_empty() && seen.insert((m.input.clone(), m.stream_id.clone())) + }); + } + for m in &mut self.excluded { + m.input = m.input.trim().to_string(); + m.stream_id = m.stream_id.trim().to_string(); + } + let mut seen = HashSet::new(); + self.excluded.retain(|m| { + !m.input.is_empty() && !m.stream_id.is_empty() && seen.insert((m.input.clone(), m.stream_id.clone())) + }); + } +} diff --git a/shared/src/model/config/target.rs b/shared/src/model/config/target.rs index 834141693..7f5700c0e 100644 --- a/shared/src/model/config/target.rs +++ b/shared/src/model/config/target.rs @@ -243,6 +243,9 @@ impl ConfigTargetDto { if let Some(sort) = self.sort.as_mut() { sort.prepare(templates)?; } + if let Some(options) = &mut self.options { + options.prepare()?; + } Ok(()) } } @@ -250,9 +253,14 @@ impl ConfigTargetDto { #[cfg(test)] mod tests; +mod channel_groups; mod filter; mod options; mod output; +pub use channel_groups::{ + ChannelGroupPolicy, ChannelGroupSelectionConfig, ChannelGroupsConfig, ChannelMatchBy, ChannelMatchNormalize, + EpgConflictMode, RuntimeAudioLanguage, +}; pub use filter::ConfigTargetFilterDto; pub use options::{ ConfigTargetOptions, ConfigTargetShareLiveStreams, DeduplicateConfig, DeduplicateKeep, DeduplicateMatchBy, diff --git a/shared/src/model/config/target/channel_groups.rs b/shared/src/model/config/target/channel_groups.rs new file mode 100644 index 000000000..c2ef1bcca --- /dev/null +++ b/shared/src/model/config/target/channel_groups.rs @@ -0,0 +1,276 @@ +use crate::error::TuliproxError; +use regex::Regex; +use std::{collections::BTreeMap, sync::Arc}; + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChannelMatchBy { + #[default] + Caption, + Name, + Title, + EpgChannelId, +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChannelGroupPolicy { + #[default] + Auto, + Priority, +} + +const fn default_true() -> bool { true } +const fn default_min_members() -> u16 { 2 } + +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct ChannelGroupSelectionConfig { + pub circuit_failures: u8, + pub circuit_cooldown_secs: u64, + pub circuit_cooldown_max_secs: u64, + pub input_circuit_failures: u8, + pub input_failure_window_secs: u64, + pub startup_budget_secs: u64, + pub max_start_attempts: u8, + pub same_member_retries: u8, + pub stall_timeout_secs: u64, + /// 0 disables the underrun trigger. + pub underrun_ratio_pct: u8, + pub underrun_window_secs: u64, + pub runtime_switch_budget_secs: u64, + pub stable_reset_secs: u64, + pub flap_window_secs: u64, + /// 0 disables the flap guard. + pub flap_max_switches: u8, + pub runtime_audio_language: RuntimeAudioLanguage, + pub early_start_response: bool, +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAudioLanguage { + #[default] + Prefer, + Require, +} + +impl Default for ChannelGroupSelectionConfig { + fn default() -> Self { + Self { + circuit_failures: 2, + circuit_cooldown_secs: 60, + circuit_cooldown_max_secs: 900, + input_circuit_failures: 3, + input_failure_window_secs: 120, + startup_budget_secs: 15, + max_start_attempts: 4, + same_member_retries: 1, + stall_timeout_secs: 8, + underrun_ratio_pct: 90, + underrun_window_secs: 15, + runtime_switch_budget_secs: 20, + stable_reset_secs: 30, + flap_window_secs: 600, + flap_max_switches: 6, + runtime_audio_language: RuntimeAudioLanguage::Prefer, + early_start_response: false, + } + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EpgConflictMode { + #[default] + Split, + Ignore, +} + +/// Grouping-only match key extensions (design §4.1); `deduplicate` never uses them. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct ChannelMatchNormalize { + #[serde(skip_serializing_if = "Option::is_none")] + pub strip_regex: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub ignore_tokens: Vec, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub aliases: BTreeMap, + #[serde(skip)] + pub t_strip_regex: Option>, +} + +impl PartialEq for ChannelMatchNormalize { + fn eq(&self, other: &Self) -> bool { + self.strip_regex == other.strip_regex + && self.ignore_tokens == other.ignore_tokens + && self.aliases == other.aliases + } +} + +impl Eq for ChannelMatchNormalize {} + +impl ChannelMatchNormalize { + pub fn is_empty(&self) -> bool { + self.strip_regex.is_none() && self.ignore_tokens.is_empty() && self.aliases.is_empty() + } + + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + self.t_strip_regex = self + .strip_regex + .as_deref() + .map(|pattern| { + Regex::new(pattern).map(Arc::new).map_err(|err| { + TuliproxError::RegexCompile(format!("channel_groups.match_normalize {pattern} {err}")) + }) + }) + .transpose()?; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ChannelGroupsConfig { + #[serde(default)] + pub enabled: bool, + #[serde(default)] + pub match_by: ChannelMatchBy, + #[serde(default)] + pub match_as_ascii: bool, + #[serde(default, skip_serializing_if = "ChannelMatchNormalize::is_empty")] + pub match_normalize: ChannelMatchNormalize, + #[serde(default)] + pub epg_conflict: EpgConflictMode, + #[serde(default)] + pub name_strip_quality: bool, + #[serde(default = "default_min_members")] + pub min_members: u16, + #[serde(default)] + pub policy_default: ChannelGroupPolicy, + #[serde(default = "default_true")] + pub allow_provider_grace: bool, + #[serde(default)] + pub selection: ChannelGroupSelectionConfig, +} + +impl Default for ChannelGroupsConfig { + fn default() -> Self { + Self { + enabled: false, + match_by: ChannelMatchBy::default(), + match_as_ascii: false, + match_normalize: ChannelMatchNormalize::default(), + epg_conflict: EpgConflictMode::default(), + name_strip_quality: false, + min_members: default_min_members(), + policy_default: ChannelGroupPolicy::default(), + allow_provider_grace: default_true(), + selection: ChannelGroupSelectionConfig::default(), + } + } +} + +impl ChannelGroupsConfig { + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + if self.min_members < 2 { + return Err(TuliproxError::ConfigTarget("`channel_groups.min_members` must be at least 2".to_string())); + } + if self.selection.max_start_attempts == 0 || self.selection.circuit_failures == 0 { + return Err(TuliproxError::ConfigTarget( + "`channel_groups.selection` attempts and failure thresholds must be > 0".to_string(), + )); + } + if self.selection.input_circuit_failures < 2 { + return Err(TuliproxError::ConfigTarget( + "`channel_groups.selection.input_circuit_failures` must be at least 2".to_string(), + )); + } + if self.selection.underrun_ratio_pct > 100 { + return Err(TuliproxError::ConfigTarget( + "`channel_groups.selection.underrun_ratio_pct` must be between 0 and 100".to_string(), + )); + } + if self.selection.circuit_cooldown_max_secs < self.selection.circuit_cooldown_secs { + self.selection.circuit_cooldown_max_secs = self.selection.circuit_cooldown_secs; + } + self.match_normalize.prepare() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_match_design() { + let cfg: ChannelGroupsConfig = serde_json::from_str(r#"{"enabled":true}"#).expect("parses"); + assert!(cfg.enabled); + assert_eq!(cfg.match_by, ChannelMatchBy::Caption); + assert!(!cfg.match_as_ascii); + assert_eq!(cfg.min_members, 2); + assert_eq!(cfg.policy_default, ChannelGroupPolicy::Auto); + assert!(cfg.allow_provider_grace); + assert_eq!(cfg.selection, ChannelGroupSelectionConfig::default()); + assert_eq!(cfg.selection.circuit_failures, 2); + assert_eq!(cfg.selection.circuit_cooldown_secs, 60); + assert_eq!(cfg.selection.circuit_cooldown_max_secs, 900); + assert_eq!(cfg.selection.startup_budget_secs, 15); + assert_eq!(cfg.selection.max_start_attempts, 4); + assert_eq!(cfg.selection.same_member_retries, 1); + assert_eq!(cfg.selection.stall_timeout_secs, 8); + assert_eq!(cfg.selection.runtime_switch_budget_secs, 20); + assert_eq!(cfg.selection.input_circuit_failures, 3); + assert_eq!(cfg.selection.input_failure_window_secs, 120); + assert_eq!(cfg.selection.underrun_ratio_pct, 90); + assert_eq!(cfg.selection.underrun_window_secs, 15); + assert_eq!(cfg.selection.stable_reset_secs, 30); + assert_eq!(cfg.selection.flap_window_secs, 600); + assert_eq!(cfg.selection.flap_max_switches, 6); + assert_eq!(cfg.match_normalize, ChannelMatchNormalize::default()); + assert_eq!(cfg.epg_conflict, EpgConflictMode::Split); + assert!(!cfg.name_strip_quality); + assert_eq!(cfg.selection.runtime_audio_language, RuntimeAudioLanguage::Prefer); + assert!(!cfg.selection.early_start_response); + } + + #[test] + fn match_normalize_compiles_regex_on_prepare() { + let mut cfg: ChannelGroupsConfig = serde_json::from_str( + r#"{"enabled":true,"match_normalize":{"strip_regex":"^[A-Z]{2}:\\s*","ignore_tokens":["hevc"],"aliases":{"das erste":"ard"}}}"#, + ) + .expect("parses"); + cfg.prepare().expect("valid"); + assert!(cfg.match_normalize.t_strip_regex.is_some()); + let mut bad: ChannelGroupsConfig = + serde_json::from_str(r#"{"enabled":true,"match_normalize":{"strip_regex":"("}}"#).expect("parses"); + assert!(bad.prepare().is_err()); + } + + #[test] + fn rejects_out_of_range_selection_values() { + let mut ratio: ChannelGroupsConfig = + serde_json::from_str(r#"{"enabled":true,"selection":{"underrun_ratio_pct":101}}"#).expect("parses"); + assert!(ratio.prepare().is_err()); + let mut input: ChannelGroupsConfig = + serde_json::from_str(r#"{"enabled":true,"selection":{"input_circuit_failures":1}}"#).expect("parses"); + assert!(input.prepare().is_err()); + } + + #[test] + fn rejects_unknown_fields_and_min_members_below_two() { + assert!(serde_json::from_str::(r#"{"enabled":true,"weight_qos":5}"#).is_err()); + let mut cfg: ChannelGroupsConfig = serde_json::from_str(r#"{"enabled":true,"min_members":1}"#).expect("parses"); + assert!(cfg.prepare().is_err()); + } + + #[test] + fn epg_channel_id_match_by_parses() { + let cfg: ChannelGroupsConfig = + serde_json::from_str(r#"{"enabled":true,"match_by":"epg_channel_id","policy_default":"priority"}"#) + .expect("parses"); + assert_eq!(cfg.match_by, ChannelMatchBy::EpgChannelId); + assert_eq!(cfg.policy_default, ChannelGroupPolicy::Priority); + } +} diff --git a/shared/src/model/config/target/options.rs b/shared/src/model/config/target/options.rs index a4b8cb84f..33901fca9 100644 --- a/shared/src/model/config/target/options.rs +++ b/shared/src/model/config/target/options.rs @@ -122,6 +122,8 @@ pub struct ConfigTargetOptions { pub epg_output: EpgOutputOptions, #[serde(default, skip_serializing_if = "Option::is_none")] pub force_redirect: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub channel_groups: Option, } impl ConfigTargetOptions { @@ -132,6 +134,14 @@ impl ConfigTargetOptions { && self.deduplicate.is_none() && self.epg_output.is_empty() && self.force_redirect.is_none_or(|f| f.has_full_flags() || f.is_empty()) + && self.channel_groups.is_none() + } + + pub fn prepare(&mut self) -> Result<(), crate::error::TuliproxError> { + if let Some(channel_groups) = &mut self.channel_groups { + channel_groups.prepare()?; + } + Ok(()) } pub const fn lowercase_epg_ids(&self) -> bool { self.epg_output.lowercase_ids } diff --git a/shared/src/model/event/kind.rs b/shared/src/model/event/kind.rs index 519b284fc..4886bf676 100644 --- a/shared/src/model/event/kind.rs +++ b/shared/src/model/event/kind.rs @@ -62,6 +62,9 @@ pub enum EventKind { RecordingProgress, /// Appended last for the same reason. MaintenanceChanged, + ChannelGroupMemberSelected, + ChannelGroupExhausted, + ChannelGroupCircuitChanged, } impl EventKind { @@ -69,7 +72,7 @@ impl EventKind { /// /// The mask type below indexes into this, so the order is load-bearing: /// it is the bit order, not just a listing. - pub const ALL: [Self; 43] = [ + pub const ALL: [Self; 46] = [ Self::ServerError, Self::ServerStarted, Self::ServerShutdown, @@ -113,6 +116,9 @@ impl EventKind { Self::AuthPermissionDenied, Self::RecordingProgress, Self::MaintenanceChanged, + Self::ChannelGroupMemberSelected, + Self::ChannelGroupExhausted, + Self::ChannelGroupCircuitChanged, ]; /// This kind's bit position. @@ -177,7 +183,10 @@ impl EventKind { // Sits with the metadata-update events it is produced by. | Self::StreamProbeFailed | Self::NotificationDeadLettered - | Self::ScheduledTaskFailed => Permission::SystemRead, + | Self::ScheduledTaskFailed + | Self::ChannelGroupMemberSelected + | Self::ChannelGroupExhausted + | Self::ChannelGroupCircuitChanged => Permission::SystemRead, Self::MaintenanceChanged => Permission::MaintenanceRead, } } @@ -278,6 +287,9 @@ impl EventKind { Self::AuthSignInThrottled => "auth.sign_in.throttled", Self::AuthPermissionDenied => "auth.permission.denied", Self::MaintenanceChanged => "maintenance.changed", + Self::ChannelGroupMemberSelected => "channel_group.member_selected", + Self::ChannelGroupExhausted => "channel_group.exhausted", + Self::ChannelGroupCircuitChanged => "channel_group.circuit_changed", } } diff --git a/shared/src/model/event/message.rs b/shared/src/model/event/message.rs index f584d06a1..2fdaa4843 100644 --- a/shared/src/model/event/message.rs +++ b/shared/src/model/event/message.rs @@ -3,12 +3,13 @@ use crate::model::{ auth_audit::{AuthAuditEvent, AuthAuditOutcome}, notification::{registry, EventId, Severity}, stats::SourceStats, - ActiveUserConnectionChange, ConfigReloadFailure, ConfigType, ConnectionDenied, DiskAlert, LibraryScanProgressEvent, - LibraryScanSummaryStatus, MaintenanceEntries, MetadataUpdateFailure, MsgKind, NotificationDeadLetter, Permission, - PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, PlaylistUpdateState, - ProviderAccountEvent, ProviderAccountState, ProviderFetchFailure, ProviderPoolExhausted, ProviderPriorityFallback, - RecordingLifecycleMessage, ScheduledTaskFailure, ServerLifecycleEvent, ServerLifecycleState, StreamProbeFailure, - SystemInfo, UserLifecycleEvent, UserLifecycleState, WatchChanges, WatchDisabled, WatchUnmatched, + ActiveUserConnectionChange, CapacityTier, CircuitState, ConfigReloadFailure, ConfigType, ConnectionDenied, + DiskAlert, LibraryScanProgressEvent, LibraryScanSummaryStatus, MaintenanceEntries, MetadataUpdateFailure, MsgKind, + NotificationDeadLetter, Permission, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, + PlaylistUpdateState, ProviderAccountEvent, ProviderAccountState, ProviderFetchFailure, ProviderPoolExhausted, + ProviderPriorityFallback, RecordingLifecycleMessage, ScheduledTaskFailure, ServerLifecycleEvent, + ServerLifecycleState, StreamProbeFailure, SystemInfo, UserLifecycleEvent, UserLifecycleState, WatchChanges, + WatchDisabled, WatchUnmatched, }; use std::sync::Arc; @@ -100,6 +101,22 @@ pub enum EventMessage { /// see [`EventMessage::kind`] - so a subscriber can ask for the failures /// without being woken by every successful sign-in. AuthAudit(AuthAuditEvent), + + ChannelGroupMemberSelected { + target_id: u16, + group_vid: u32, + member_vid: u32, + tier: CapacityTier, + }, + ChannelGroupExhausted { + target_id: u16, + group_vid: u32, + }, + ChannelGroupCircuitChanged { + input_name: Arc, + input_stream_id: Arc, + state: CircuitState, + }, } impl EventMessage { @@ -170,6 +187,9 @@ impl EventMessage { AuthAuditOutcome::SignInThrottled => EventKind::AuthSignInThrottled, AuthAuditOutcome::PermissionDenied => EventKind::AuthPermissionDenied, }, + Self::ChannelGroupMemberSelected { .. } => EventKind::ChannelGroupMemberSelected, + Self::ChannelGroupExhausted { .. } => EventKind::ChannelGroupExhausted, + Self::ChannelGroupCircuitChanged { .. } => EventKind::ChannelGroupCircuitChanged, } } @@ -267,7 +287,12 @@ impl EventMessage { Self::RecordingChanged => registry::RECORDING_QUEUE_CHANGED, Self::RecordingRulesChanged => registry::RECORDING_RULES_CHANGED, Self::MaintenanceChanged(_) => registry::MAINTENANCE_CHANGED, - Self::PlaylistUpdateProgress(_) | Self::SystemInfoUpdate(_) | Self::RecordingProgress => return None, + Self::PlaylistUpdateProgress(_) + | Self::SystemInfoUpdate(_) + | Self::RecordingProgress + | Self::ChannelGroupMemberSelected { .. } + | Self::ChannelGroupExhausted { .. } + | Self::ChannelGroupCircuitChanged { .. } => return None, }) } @@ -323,6 +348,32 @@ impl EventMessage { Self::NotificationDeadLettered(dead_letter) => encode(dead_letter), Self::ScheduledTaskFailed(failure) => encode(failure), Self::AuthAudit(event) => encode(event), + Self::ChannelGroupMemberSelected { target_id, group_vid, member_vid, tier } => { + serde_json::json!({ + "target_id": target_id, + "group_vid": group_vid, + "member_vid": member_vid, + "tier": tier, + }) + } + Self::ChannelGroupExhausted { target_id, group_vid } => { + serde_json::json!({ + "target_id": target_id, + "group_vid": group_vid, + }) + } + Self::ChannelGroupCircuitChanged { input_name, input_stream_id, state } => { + let state_str = match state { + CircuitState::Closed => "closed", + CircuitState::OpenUntil(_) => "open", + CircuitState::HalfOpenInFlight => "half_open", + }; + serde_json::json!({ + "input_name": input_name.as_ref(), + "input_stream_id": input_stream_id.as_ref(), + "state": state_str, + }) + } } } diff --git a/shared/src/model/event/sink.rs b/shared/src/model/event/sink.rs index 8fb68515c..a399eec32 100644 --- a/shared/src/model/event/sink.rs +++ b/shared/src/model/event/sink.rs @@ -32,6 +32,16 @@ pub trait EventSink: Send + Sync { _exp_date: Option, ) { } + /// Forward probe technical metadata to the member health index. + fn observe_stream_probe( + &self, + _input_name: Arc, + _input_stream_id: Arc, + _video: Option<&str>, + _audio: Option<&str>, + _bitrate_kbps: u32, + ) { + } } /// The sink that drops everything. @@ -57,6 +67,16 @@ impl EventSink for Arc { ) { (**self).observe_provider_account(name, identity, status, exp_date); } + fn observe_stream_probe( + &self, + input_name: Arc, + input_stream_id: Arc, + video: Option<&str>, + audio: Option<&str>, + bitrate_kbps: u32, + ) { + (**self).observe_stream_probe(input_name, input_stream_id, video, audio, bitrate_kbps); + } } impl EventSink for &T { diff --git a/shared/src/model/item_field.rs b/shared/src/model/item_field.rs index c78e1fff5..f33c061c1 100644 --- a/shared/src/model/item_field.rs +++ b/shared/src/model/item_field.rs @@ -79,6 +79,7 @@ pub enum HeaderField { EpgChannelId, Chno, Genre, + ChannelGroup, } impl HeaderField { @@ -110,6 +111,7 @@ impl HeaderField { ("epg_id", HeaderField::EpgChannelId), ("chno", HeaderField::Chno), ("genre", HeaderField::Genre), + ("channel_group", HeaderField::ChannelGroup), ]; TABLE.iter().find(|(key, _)| name.eq_ignore_ascii_case(key)).map(|(_, field)| *field) } @@ -135,6 +137,7 @@ impl HeaderField { Self::EpgChannelId => "epg_channel_id", Self::Chno => "chno", Self::Genre => "genre", + Self::ChannelGroup => "channel_group", } } } diff --git a/shared/src/model/mapping.rs b/shared/src/model/mapping.rs index df0d3fb30..ab4c70806 100644 --- a/shared/src/model/mapping.rs +++ b/shared/src/model/mapping.rs @@ -31,6 +31,7 @@ pub const MAPPER_FIELDS: &[HeaderField] = &[ HeaderField::Rec, HeaderField::Url, HeaderField::EpgChannelId, + HeaderField::ChannelGroup, ]; /// Whether `name` resolves to one of `allowed`. diff --git a/shared/src/model/media_properties.rs b/shared/src/model/media_properties.rs index ca06c9cd0..a2fef7027 100644 --- a/shared/src/model/media_properties.rs +++ b/shared/src/model/media_properties.rs @@ -167,10 +167,15 @@ impl MediaQuality { fn from_ffprobe_info_audio(audio: Option<&str>) -> Option<(AudioCodec, AudioChannels)> { // Assuming the first audio stream is the primary one. - let audio_info = audio.and_then(|v| serde_json::from_str::>(v).ok())?; + let audio_val = audio.and_then(|v| serde_json::from_str::(v).ok())?; + let audio_info = match &audio_val { + Value::Object(map) => Some(map), + Value::Array(arr) => arr.first().and_then(Value::as_object), + _ => None, + }?; // Audio codec - let audio_codec = get_value(&audio_info, &["codec_name"]) + let audio_codec = get_value(audio_info, &["codec_name"]) .and_then(|v| v.as_str().map(str::to_lowercase)) .map_or(AudioCodec::default(), |name| match name.as_str() { "aac" => AudioCodec::AAC, @@ -183,16 +188,16 @@ impl MediaQuality { }); // Audio channels - let audio_channels = - get_value(&audio_info, &["channels"]).and_then(|v| v.as_i64()).map_or(AudioChannels::default(), |ch| { - match ch { - 8 => AudioChannels::Surround71, - 6 => AudioChannels::Surround51, - 2 => AudioChannels::Stereo, - 1 => AudioChannels::Mono, - _ => AudioChannels::default(), - } - }); + let audio_channels = get_value(audio_info, &["channels"]).and_then(|v| v.as_i64()).map_or( + AudioChannels::default(), + |ch| match ch { + 8 => AudioChannels::Surround71, + 6 => AudioChannels::Surround51, + 2 => AudioChannels::Stereo, + 1 => AudioChannels::Mono, + _ => AudioChannels::default(), + }, + ); Some((audio_codec, audio_channels)) } @@ -200,15 +205,20 @@ impl MediaQuality { fn from_ffprobe_info_video( video: Option<&str>, ) -> Option<(VideoResolution, VideoCodec, VideoDynamicRange, VideoBitDepth)> { - let video_info = video.and_then(|v| serde_json::from_str::>(v).ok())?; + let video_val = video.and_then(|v| serde_json::from_str::(v).ok())?; + let video_info = match &video_val { + Value::Object(map) => Some(map), + Value::Array(arr) => arr.first().and_then(Value::as_object), + _ => None, + }?; // 1. Classify video resolution from the frame dimensions - let width = get_value(&video_info, &["width", "coded_width"]).and_then(|v| v.as_u64()); - let height = get_value(&video_info, &["height", "coded_height"]).and_then(|v| v.as_u64()); + let width = get_value(video_info, &["width", "coded_width"]).and_then(|v| v.as_u64()); + let height = get_value(video_info, &["height", "coded_height"]).and_then(|v| v.as_u64()); let resolution = VideoResolution::from_dimensions(width, height); // 2. Classify video codec - let video_codec = get_value(&video_info, &["codec_name"]) + let video_codec = get_value(video_info, &["codec_name"]) .and_then(|v| v.as_str().map(str::to_lowercase)) .map_or(VideoCodec::default(), |name| match name.as_str() { "h264" => VideoCodec::H264, @@ -222,12 +232,12 @@ impl MediaQuality { // 3. Classify dynamic range let dynamic_range = { let tag_string = - get_value(&video_info, &["codec_tag_string"]).and_then(|v| v.as_str().map(str::to_lowercase)); + get_value(video_info, &["codec_tag_string"]).and_then(|v| v.as_str().map(str::to_lowercase)); if tag_string == Some("dovi".to_string()) { VideoDynamicRange::DV } else { - get_value(&video_info, &["color_transfer"]).and_then(|v| v.as_str().map(str::to_lowercase)).map_or( + get_value(video_info, &["color_transfer"]).and_then(|v| v.as_str().map(str::to_lowercase)).map_or( VideoDynamicRange::SDR, |ct| match ct.as_str() { "smpte2084" => VideoDynamicRange::HDR, // Generic HDR/HDR10 @@ -239,7 +249,7 @@ impl MediaQuality { }; // 4. Classify bit depth - let bit_depth = get_value(&video_info, &["pix_fmt"]).and_then(|v| v.as_str().map(ToString::to_string)).map_or( + let bit_depth = get_value(video_info, &["pix_fmt"]).and_then(|v| v.as_str().map(ToString::to_string)).map_or( VideoBitDepth::Eight, |fmt| { if fmt.contains("10le") || fmt.contains("10be") { diff --git a/shared/src/model/mod.rs b/shared/src/model/mod.rs index ccc549d1e..c5a8f731e 100644 --- a/shared/src/model/mod.rs +++ b/shared/src/model/mod.rs @@ -1,6 +1,7 @@ mod active_user_connection_change; mod auth; mod auth_audit; +pub mod channel_group; mod cluster_flags; mod config; mod connection_denied; @@ -78,18 +79,18 @@ mod xtream; pub mod xtream_const; pub use self::{ - active_user_connection_change::*, auth::*, auth_audit::*, cluster_flags::*, config::*, connection_denied::*, - custom_video_stream_type::*, epg::*, epg_grid::*, epg_request::*, event::*, input_refresh_policy::*, - input_update_action::*, ip_check::*, item_field::*, library_request::*, log::*, maintenance::*, mapping::*, - media_properties::*, messaging::*, metadata_update_failure::*, notification::*, notification_dead_letter::*, - pagination::*, playlist::*, playlist_categories::*, playlist_info_document::*, playlist_request::*, - playlist_update_run::*, playlist_update_status::*, processing_order::*, progress::*, provider_fetch_failure::*, - provider_pool::*, recording::*, recording_math::*, regex_cache::*, resource::*, scheduled_task_failure::*, - search_fields::*, search_request::*, server_lifecycle::*, short_epg::*, stalker::*, stalker_item::*, stats::*, - status_check::*, stream_history::*, stream_history_record::*, stream_info::*, stream_meter::*, - stream_probe_failure::*, stream_properties::*, strm_export_style::*, system_info::*, target_bouquet::*, - target_type::*, transfer::*, ui_playlist_item::*, user_command::*, user_id::*, user_lifecycle::*, uuidtype::*, - watch_health::*, web_socket::*, xtream::*, + active_user_connection_change::*, auth::*, auth_audit::*, channel_group::*, cluster_flags::*, config::*, + connection_denied::*, custom_video_stream_type::*, epg::*, epg_grid::*, epg_request::*, event::*, + input_refresh_policy::*, input_update_action::*, ip_check::*, item_field::*, library_request::*, log::*, + maintenance::*, mapping::*, media_properties::*, messaging::*, metadata_update_failure::*, notification::*, + notification_dead_letter::*, pagination::*, playlist::*, playlist_categories::*, playlist_info_document::*, + playlist_request::*, playlist_update_run::*, playlist_update_status::*, processing_order::*, progress::*, + provider_fetch_failure::*, provider_pool::*, recording::*, recording_math::*, regex_cache::*, resource::*, + scheduled_task_failure::*, search_fields::*, search_request::*, server_lifecycle::*, short_epg::*, stalker::*, + stalker_item::*, stats::*, status_check::*, stream_history::*, stream_history_record::*, stream_info::*, + stream_meter::*, stream_probe_failure::*, stream_properties::*, strm_export_style::*, system_info::*, + target_bouquet::*, target_type::*, transfer::*, ui_playlist_item::*, user_command::*, user_id::*, + user_lifecycle::*, uuidtype::*, watch_health::*, web_socket::*, xtream::*, }; pub use ids::*; pub use prepare::*; diff --git a/shared/src/model/playlist/header.rs b/shared/src/model/playlist/header.rs index e2ab44f17..17e1d31c0 100644 --- a/shared/src/model/playlist/header.rs +++ b/shared/src/model/playlist/header.rs @@ -56,6 +56,12 @@ pub struct PlaylistItemHeader { pub input_stream_id: Arc, #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] pub upstream_user_agent: Option>, + /// Mapping override for channel grouping: `Some("")` opts out, `Some(key)` forces a key. + #[serde(default, with = "arc_str_option_serde")] + pub channel_group: Option>, + /// Set on group items by the grouping stage; copies (favourites) inherit it. + #[serde(default)] + pub channel_group_uuid: Option, } impl Default for PlaylistItemHeader { @@ -84,6 +90,8 @@ impl Default for PlaylistItemHeader { source_ordinal: 0, input_stream_id: "".intern(), upstream_user_agent: None, + channel_group: None, + channel_group_uuid: None, } } } @@ -166,6 +174,7 @@ impl crate::model::FieldGet for crate::model::PlaylistItemHeader { HeaderField::Genre => { self.additional_properties.as_ref().and_then(StreamProperties::genre).map(FieldRef::Shared) } + HeaderField::ChannelGroup => self.channel_group.as_ref().map(FieldRef::Shared), // Not carried by the header. HeaderField::ProviderId => None, } @@ -192,6 +201,7 @@ impl crate::model::FieldSet for crate::model::PlaylistItemHeader { self.name = interned; } HeaderField::EpgChannelId => self.epg_channel_id = Some(value.intern()), + HeaderField::ChannelGroup => self.channel_group = Some(value.intern()), HeaderField::Chno => match value.parse::() { Ok(parsed) => self.chno = parsed, Err(_) => return false, diff --git a/shared/src/model/playlist/item.rs b/shared/src/model/playlist/item.rs index a6e5c9dc6..ac7bf214b 100644 --- a/shared/src/model/playlist/item.rs +++ b/shared/src/model/playlist/item.rs @@ -273,6 +273,8 @@ impl From<&XtreamPlaylistItem> for PlaylistItem { source_ordinal: item.source_ordinal, input_stream_id: input_stream_id.unwrap_or_else(|| "".intern()), upstream_user_agent: item.upstream_user_agent.clone(), + channel_group: None, + channel_group_uuid: None, }; PlaylistItem { header } @@ -305,6 +307,8 @@ impl From<&M3uPlaylistItem> for PlaylistItem { source_ordinal: item.source_ordinal, input_stream_id: item.get_input_stream_id().unwrap_or_else(|| "".intern()), upstream_user_agent: item.upstream_user_agent.clone(), + channel_group: None, + channel_group_uuid: None, }; PlaylistItem { header } @@ -375,6 +379,8 @@ impl PlaylistItem { source_ordinal: 0, input_stream_id: stream_id_str, upstream_user_agent: None, + channel_group: None, + channel_group_uuid: None, }; PlaylistItem { header } diff --git a/shared/src/model/playlist/m3u.rs b/shared/src/model/playlist/m3u.rs index 53150affc..82f95d98c 100644 --- a/shared/src/model/playlist/m3u.rs +++ b/shared/src/model/playlist/m3u.rs @@ -301,7 +301,11 @@ impl crate::model::FieldGet for M3uPlaylistItem { // M3U resource endpoint resolves a URL path segment through // `get_field`, so making these resolvable would turn a 404 into a // response. Preserved from the string-keyed accessor this replaces. - HeaderField::Id | HeaderField::Input | HeaderField::Type | HeaderField::Genre => None, + HeaderField::Id + | HeaderField::Input + | HeaderField::Type + | HeaderField::Genre + | HeaderField::ChannelGroup => None, } } } diff --git a/shared/src/model/playlist/tests/identity.rs b/shared/src/model/playlist/tests/identity.rs index 51de82924..6d9be5e6c 100644 --- a/shared/src/model/playlist/tests/identity.rs +++ b/shared/src/model/playlist/tests/identity.rs @@ -177,3 +177,24 @@ fn legacy_playlist_items_fall_back_to_provider_id_without_virtual_id() { assert_eq!(identified_xtream_item.input_stream_id.as_ref(), "explicit-origin-alpha"); assert_eq!(identified_xtream_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); } + +#[test] +fn channel_group_field_round_trips_through_dsl_setters() { + use crate::model::{FieldGet, FieldSet, HeaderField, PlaylistItemHeader}; + let mut header = PlaylistItemHeader::default(); + assert_eq!(HeaderField::parse("channel_group"), Some(HeaderField::ChannelGroup)); + assert_eq!(HeaderField::ChannelGroup.as_str(), "channel_group"); + assert!(header.set(HeaderField::ChannelGroup, "ZDF")); + assert_eq!(header.channel_group.as_deref(), Some("ZDF")); + assert!(header.set(HeaderField::ChannelGroup, "")); + assert_eq!(header.channel_group.as_deref(), Some("")); + assert!(header.get(HeaderField::ChannelGroup).is_some()); +} + +#[test] +fn header_without_channel_group_field_still_deserializes() { + let mut value = serde_json::to_value(crate::model::PlaylistItemHeader::default()).expect("serializes"); + value.as_object_mut().expect("object").remove("channel_group"); + let header: crate::model::PlaylistItemHeader = serde_json::from_value(value).expect("deserializes"); + assert!(header.channel_group.is_none()); +} diff --git a/shared/src/model/playlist/tests/m3u.rs b/shared/src/model/playlist/tests/m3u.rs index bf46a90e7..4ba5cdb26 100644 --- a/shared/src/model/playlist/tests/m3u.rs +++ b/shared/src/model/playlist/tests/m3u.rs @@ -21,6 +21,7 @@ fn header_field_parse_round_trips_every_variant() { HeaderField::EpgChannelId, HeaderField::Chno, HeaderField::Genre, + HeaderField::ChannelGroup, ] { assert_eq!(HeaderField::parse(field.as_str()), Some(field), "{field} did not round-trip"); // Lookup stayed case-insensitive. @@ -45,7 +46,7 @@ fn m3u_item_exposes_provider_id_but_not_the_header_only_fields() { assert_eq!(item.get_field("chno").as_deref(), Some("7")); assert_eq!(item.get_field("caption").as_deref(), Some("Channel")); - for absent in ["id", "input", "type", "genre", "not_a_field"] { + for absent in ["id", "input", "type", "genre", "channel_group", "not_a_field"] { assert!(item.get_field(absent).is_none(), "{absent} must not resolve on an M3U item"); } } diff --git a/shared/src/model/stream_info.rs b/shared/src/model/stream_info.rs index f09a49ade..3c535d95d 100644 --- a/shared/src/model/stream_info.rs +++ b/shared/src/model/stream_info.rs @@ -38,10 +38,12 @@ impl StreamTechnicalInfo { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] pub struct StreamChannel { pub target_id: u16, pub virtual_id: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group_virtual_id: Option, pub provider_id: u32, #[serde(with = "arc_str_serde")] pub input_name: Arc, @@ -71,6 +73,9 @@ pub struct StreamChannel { } impl StreamChannel { + #[must_use] + pub fn client_virtual_id(&self) -> u32 { self.group_virtual_id.unwrap_or(self.virtual_id) } + /// Returns a copy of this channel with the supplied EPG reference timestamp /// installed. Used by the M3U non-HLS path to thread an archive/catchup /// timestamp parsed from the request URL into the `StreamChannel` that the @@ -99,6 +104,7 @@ impl XtreamPlaylistItem { StreamChannel { target_id, virtual_id: self.virtual_id.get(), + group_virtual_id: None, provider_id: self.provider_id, input_name: Arc::clone(&self.input_name), item_type: self.item_type, @@ -123,6 +129,7 @@ impl M3uPlaylistItem { StreamChannel { target_id, virtual_id: self.virtual_id.get(), + group_virtual_id: None, provider_id: self.get_provider_id().unwrap_or_default(), input_name: Arc::clone(&self.input_name), item_type: self.item_type,