diff --git a/CHANGELOG.md b/CHANGELOG.md index a0377adc3..5381e74e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ## ⚠️ Breaking Changes +- **Web UI sessions must sign in again once.** The new maintenance permissions bump the permission schema, so tokens + issued before the update are rejected with `X-Token-Refresh: required` and the Web UI returns to the sign-in page. + - **`playlist.watch.changed` carries every watch change of one update.** The payload is now `{ target, changes: [...] }`, where each change has a `kind` (`groups_added`, `groups_removed`, `channels_added`, `channels_removed`). Templates using `{{watch.group}}`, `{{watch.added}}` or `{{watch.removed}}` must be updated: @@ -183,6 +186,12 @@ ## 🌟 New Features +- **Maintenance mode.** Inputs can be put into maintenance from the new dashboard card or through + `/api/v1/maintenance` (`*` addresses all inputs). New streams of an input in maintenance get the new + `maintenance.ts` custom video; playlist and EPG updates keep the last persisted state, and probes, metadata + resolution, Stalker refreshes and DVR starts do not contact the provider. Running streams and recordings continue. + The state is stored in `maintenance.json` and survives restarts. New permissions `maintenance.read` and + `maintenance.write`. See [Maintenance mode](docs/src/rest-api-cookbook.md#maintenance-mode). - **Optional fast start for shared live HLS.** `reverse_proxy.hls_cache.startup.mode` selects `conservative` (default, unchanged behavior), `first_ready` (admits playback after one processed start segment) or `progressive` (streams the raw TS start segment while the shared origin fill is still running). Progressive replay memory, diff --git a/README.md b/README.md index ea9a7fa96..4ff4e8d40 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,9 @@ Upstream providers are not always reliable. Tuliprox is designed to preserve usa - Independent inputs can update in parallel while optional sequential groups protect providers that must not be refreshed concurrently - Automatic Xtream account-expiration refresh with throttling designed to reduce unnecessary provider requests - Stalker capability knowledge is reused during a running client session to avoid repeatedly probing known-failing endpoint variants +- **Maintenance mode** takes an input out of service from the dashboard or the REST API: new streams get a + dedicated maintenance video, the last playlist and EPG keep being served, and no subsystem contacts the provider until + maintenance ends; running streams and recordings continue **Benefit:** a broken refresh or failing alias does not have to destroy an otherwise usable catalog. @@ -219,6 +222,8 @@ Tuliprox can improve incomplete provider data and make large catalogs easier to Tuliprox can be configured and monitored directly from the browser. - **Dashboard** with backend status, active streams, system metrics, and provider capacity +- **Maintenance card** on the dashboard that turns red while inputs are in maintenance, with a dialog to put inputs + into or take them out of maintenance with a remark - **Source Editor** for M3U, Xtream, Stalker, Plex, Emby, Jellyfin, local-library, and target configuration - Drag & drop, block selection, batch editing, and target bouquet whitelist/blacklist editing - **Input-focused Update view** with per-input actions, affected-target selection, status, last update time, and @@ -271,7 +276,7 @@ Expose one Tuliprox instance to multiple users without giving everyone identical #### RBAC -- 14 permissions across 7 domains +- 20 permissions across 9 domains - Independent `.read` and `.write` grants - Custom groups and multi-group membership - Additive permission resolution diff --git a/backend/app/src/api/api_utils/maintenance.rs b/backend/app/src/api/api_utils/maintenance.rs new file mode 100644 index 000000000..fb6840606 --- /dev/null +++ b/backend/app/src/api/api_utils/maintenance.rs @@ -0,0 +1,69 @@ +//! Gate 1: a new playback request for an input in maintenance gets the +//! maintenance clip instead of a provider connection. +//! +//! Requests that continue something already admitted pass: the DVR's internal +//! proxy user (the DVR gate decided before it reserved the slot) and requests +//! carrying a slot reserved for them. + +use super::admission_failure_response; +use crate::{ + api::{ + endpoints::hls_api::hls_admission_failure_manifest_response, + model::{AppState, UserSession}, + }, + auth::Fingerprint, + model::{ConfigInput, ProxyUserCredentials}, +}; +use axum::http::HeaderMap; +use shared::model::{ConnectFailureReason, StreamChannel}; +use std::sync::Arc; + +/// A session that is counted against the user's connections is a running playback; a request +/// for it continues that playback. A known but uncounted session (prepared, preserved, expired) +/// would be admitted anew. +pub(crate) fn is_running_playback(user_session: Option<&UserSession>) -> bool { + user_session.is_some_and(|session| session.lifecycle.is_counted()) +} + +/// The maintenance answer for a new playback, given before admission so it neither counts +/// against the user's connections nor evicts anything. +pub(crate) async fn maintenance_rejection_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + stream_channel: StreamChannel, + input: &ConfigInput, + req_headers: &HeaderMap, + is_hls: bool, +) -> axum::response::Response { + if is_hls { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + user, + stream_channel, + Arc::clone(&input.name), + req_headers, + ConnectFailureReason::Maintenance, + ) + .await; + } + admission_failure_response( + app_state, + fingerprint, + user, + stream_channel, + Arc::clone(&input.name), + req_headers, + ConnectFailureReason::Maintenance, + ) +} + +pub(crate) fn blocks_new_playback( + app_state: &AppState, + input: &ConfigInput, + user: &ProxyUserCredentials, + has_preacquired_handle: bool, +) -> bool { + !has_preacquired_handle && !user.is_recording_proxy_user() && app_state.maintenance.is_in_maintenance(&input.name) +} diff --git a/backend/app/src/api/api_utils/mod.rs b/backend/app/src/api/api_utils/mod.rs index 94fab5a28..d689cb798 100644 --- a/backend/app/src/api/api_utils/mod.rs +++ b/backend/app/src/api/api_utils/mod.rs @@ -240,6 +240,7 @@ mod auth; mod catchup; mod clock; mod local_files; +mod maintenance; mod media_server; mod metering; mod provider_acquire; @@ -324,6 +325,7 @@ pub(crate) use self::{ auth::{input_for_user, with_recording_headers}, catchup::get_catchup_session_ttl_secs, local_files::local_stream_response, + maintenance::{blocks_new_playback, is_running_playback, maintenance_rejection_response}, provider_acquire::{acquire_exact_provider_handle, stream_response_with_provider_handle}, provider_urls::select_provider_stream_url, resources::force_hls_resource_response, diff --git a/backend/app/src/api/api_utils/provider_acquire.rs b/backend/app/src/api/api_utils/provider_acquire.rs index 06e3e4772..b86083b3f 100644 --- a/backend/app/src/api/api_utils/provider_acquire.rs +++ b/backend/app/src/api/api_utils/provider_acquire.rs @@ -1,7 +1,8 @@ use super::{ - activate_session_before_stream_open, cleanup_failed_detected_catchup_hls, connection_priority_for_kind, - create_stream_response_details, detected_catchup_hls_response, get_session_reservation_ttl_secs, - get_stream_options, is_socket_bound_playback_session, is_stream_share_enabled, mark_response_as_uncompressed, + activate_session_before_stream_open, admission_failure_response, blocks_new_playback, + cleanup_failed_detected_catchup_hls, connection_priority_for_kind, create_stream_response_details, + detected_catchup_hls_response, get_session_reservation_ttl_secs, get_stream_options, + is_socket_bound_playback_session, is_stream_share_enabled, mark_response_as_uncompressed, no_custom_video_fallback_status, prepare_body_stream, prepare_stream_metering, probe_catchup_payload, record_connect_failed_attempt, reentry_suppressed_response, resolve_request_url_for_logging, resolve_xtream_vod_provider_url, select_provider_stream_url, should_pin_provider_for_session, @@ -304,6 +305,17 @@ pub(crate) async fn stream_response_with_provider_handle( grace_mode: Option, preacquired_provider_handle: Option, ) -> impl IntoResponse + Send { + if blocks_new_playback(app_state, input, user, preacquired_provider_handle.is_some()) { + return admission_failure_response( + app_state, + fingerprint, + user, + stream_channel, + Arc::clone(&input.name), + req_headers, + ConnectFailureReason::Maintenance, + ); + } let _transition_guard = app_state.active_users.acquire_playback_transition(&user.username, session_token).await; let request_log_stream_url = resolve_request_url_for_logging(input, stream_url); if log_enabled!(log::Level::Trace) { diff --git a/backend/app/src/api/api_utils/response.rs b/backend/app/src/api/api_utils/response.rs index f538f8a11..5ec214a3f 100644 --- a/backend/app/src/api/api_utils/response.rs +++ b/backend/app/src/api/api_utils/response.rs @@ -22,6 +22,7 @@ pub(super) fn admission_failure_video_type(reason: ConnectFailureReason) -> Opti ConnectFailureReason::UserAccountExpired => Some(CustomVideoStreamType::UserAccountExpired), ConnectFailureReason::UserConnectionsExhausted => Some(CustomVideoStreamType::UserConnectionsExhausted), ConnectFailureReason::ProviderConnectionsExhausted => Some(CustomVideoStreamType::ProviderConnectionsExhausted), + ConnectFailureReason::Maintenance => Some(CustomVideoStreamType::Maintenance), _ => None, } } diff --git a/backend/app/src/api/api_utils/stalker.rs b/backend/app/src/api/api_utils/stalker.rs index 21d548519..b909d4371 100644 --- a/backend/app/src/api/api_utils/stalker.rs +++ b/backend/app/src/api/api_utils/stalker.rs @@ -1,4 +1,9 @@ -use crate::{api::model::AppState, model::ConfigInput, processing::processor::re_resolve_stalker_url}; +use super::blocks_new_playback; +use crate::{ + api::model::AppState, + model::{ConfigInput, ProxyUserCredentials}, + processing::processor::re_resolve_stalker_url, +}; use axum::http::StatusCode; use shared::{ error::TuliproxError, @@ -46,12 +51,16 @@ pub(super) async fn re_resolve_stalker_url_singleflight( pub(crate) async fn resolve_initial_stalker_playback_url( app_state: &Arc, input: &ConfigInput, + user: &ProxyUserCredentials, provider_id: u32, cluster: XtreamCluster, item_type: PlaylistItemType, stream_url: &Arc, ) -> Result, TuliproxError> { - if !needs_initial_stalker_resolution(input.input_type, stream_url) { + // An input in maintenance is not contacted; the maintenance gate further down answers. + if !needs_initial_stalker_resolution(input.input_type, stream_url) + || blocks_new_playback(app_state, input, user, false) + { return Ok(Arc::clone(stream_url)); } re_resolve_stalker_url_singleflight(app_state, input, provider_id, stalker_stream_kind(cluster, item_type), false) diff --git a/backend/app/src/api/api_utils/strategy.rs b/backend/app/src/api/api_utils/strategy.rs index 7f5514004..79be46f23 100644 --- a/backend/app/src/api/api_utils/strategy.rs +++ b/backend/app/src/api/api_utils/strategy.rs @@ -1,6 +1,6 @@ use super::{ - get_redirect_alternative_url, get_stream_alternative_url, is_media_server_playback_url, redirect, - resolve_request_url_for_logging, RedirectParams, + blocks_new_playback, get_redirect_alternative_url, get_stream_alternative_url, is_media_server_playback_url, + redirect, resolve_request_url_for_logging, RedirectParams, }; #[cfg(test)] use super::{resolve_streaming_strategy_with_provider_handle, StreamingAcquireOptions}; @@ -120,6 +120,11 @@ pub async fn redirect_response<'a, P>( where P: PlaylistEntry, { + // An input in maintenance is never redirected to its provider; the proxied + // path that follows answers with the maintenance clip. + if blocks_new_playback(app_state, params.input, params.user, false) { + return None; + } let item_type = params.item.get_item_type(); let provider_url = params.item.get_provider_url(); if is_media_server_playback_url(params.input, provider_url.as_ref()) { diff --git a/backend/app/src/api/api_utils/tests.rs b/backend/app/src/api/api_utils/tests.rs index 7394fd0be..77a58cc05 100644 --- a/backend/app/src/api/api_utils/tests.rs +++ b/backend/app/src/api/api_utils/tests.rs @@ -7,6 +7,7 @@ mod http_load; mod http_smoke; mod http_soak; mod local_files; +mod maintenance; mod network_access; mod provider_leases; mod provider_urls; diff --git a/backend/app/src/api/api_utils/tests/cleanup.rs b/backend/app/src/api/api_utils/tests/cleanup.rs index daf204f95..f2aacb0a7 100644 --- a/backend/app/src/api/api_utils/tests/cleanup.rs +++ b/backend/app/src/api/api_utils/tests/cleanup.rs @@ -1327,6 +1327,7 @@ async fn test_channel_unavailable_fallback_does_not_register_body_owner_or_leak_ user_account_expired: None, panel_api_provisioning: None, hls_session_or_lease_expired: None, + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), }; app_state.app_config.custom_stream_response.store(Some(std::sync::Arc::new(custom_video))); diff --git a/backend/app/src/api/api_utils/tests/maintenance.rs b/backend/app/src/api/api_utils/tests/maintenance.rs new file mode 100644 index 000000000..81612f2c6 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/maintenance.rs @@ -0,0 +1,203 @@ +use super::{ + create_test_app_state_for_config, create_test_fingerprint, create_test_live_channel, + create_test_provider_app_config, create_test_provider_app_state, stream_response, +}; +use crate::{ + api::{api_utils::blocks_new_playback, model::AppState}, + model::{ConfigInput, ConfigInputAlias, ProxyUserCredentials, SourcesConfig, RECORDING_PROXY_USERNAME}, +}; +use shared::utils::Internable; +use std::sync::Arc; + +/// The body of the function starting with `signature`, up to its closing brace in column 0, +/// so an order check cannot match code of a neighbouring function. +fn function_body<'a>(source: &'a str, signature: &str) -> &'a str { + let start = source.find(signature).expect("function present"); + let body = &source[start..]; + &body[..body.find("\n}\n").expect("function end")] +} + +fn viewer() -> ProxyUserCredentials { + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + user +} + +fn provider_state() -> (Arc, Arc) { + let app_state = create_test_app_state_for_config(Arc::new(create_test_provider_app_config())); + let input = Arc::clone(&app_state.app_config.sources.load().inputs[0]); + (app_state, input) +} + +#[tokio::test] +async fn input_not_in_maintenance_is_not_gated() { + let (app_state, input) = provider_state(); + assert!(!blocks_new_playback(&app_state, &input, &viewer(), false)); +} + +#[tokio::test] +async fn input_in_maintenance_is_gated() { + let (app_state, input) = provider_state(); + app_state.maintenance.add(&input.name, None, 1).await.unwrap(); + assert!(blocks_new_playback(&app_state, &input, &viewer(), false)); +} + +#[tokio::test] +async fn preacquired_handle_is_not_gated() { + let (app_state, input) = provider_state(); + app_state.maintenance.add(&input.name, None, 1).await.unwrap(); + assert!(!blocks_new_playback(&app_state, &input, &viewer(), true)); +} + +#[tokio::test] +async fn recording_proxy_user_is_not_gated() { + let (app_state, input) = provider_state(); + app_state.maintenance.add(&input.name, None, 1).await.unwrap(); + let mut recorder = ProxyUserCredentials::default(); + recorder.username = RECORDING_PROXY_USERNAME.to_string(); + recorder.t_is_api_user = true; + assert!(recorder.is_recording_proxy_user()); + assert!(!blocks_new_playback(&app_state, &input, &recorder, false)); +} + +#[tokio::test] +async fn aliased_input_is_gated_by_canonical_name() { + let (app_state, provider) = provider_state(); + let input = Arc::new(ConfigInput { + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_1_alias".intern(), + url: "http://127.0.0.1:2".to_string(), + username: Some("user-b".to_string()), + password: Some("pass-b".to_string()), + priority: 1, + max_connections: 3, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..(*provider).clone() + }); + app_state + .app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + app_state.maintenance.add(&input.name, None, 1).await.unwrap(); + assert!(!app_state.maintenance.is_in_maintenance("provider_1_alias"), "aliases are never keys"); + assert!(blocks_new_playback(&app_state, &input, &viewer(), false)); +} + +#[test] +fn gate_runs_before_shared_stream_join() { + // A new viewer must not join a running provider stream of an input in maintenance. + let body = function_body( + include_str!("../provider_acquire.rs"), + "pub(crate) async fn stream_response_with_provider_handle(", + ); + let gate = body.find("if blocks_new_playback(").expect("gate present"); + let join = body.find("try_shared_stream_response_if_any(").expect("shared join present"); + assert!(gate < join, "maintenance gate must run before the shared stream join"); +} + +#[test] +fn redirect_defers_maintenance_inputs_to_the_gated_proxy_path() { + // Redirect mode would send the client straight to the provider; for an input in + // maintenance it must fall through to the proxied path, where gate 1 answers. + let body = function_body(include_str!("../strategy.rs"), "pub async fn redirect_response<"); + let gate = body.find("if blocks_new_playback(").expect("gate present"); + let redirect = body.find("Some(redirect(").expect("redirect response present"); + assert!(gate < redirect, "maintenance must be checked before any redirect is built"); +} + +/// One MPEG-TS packet whose payload marks it as the maintenance clip. +fn maintenance_packet() -> Vec { + let mut packet = vec![0xff_u8; 188]; + packet[0] = 0x47; + packet[1] = 0x01; + packet[2] = 0x00; + packet[3] = 0x10; + packet[4..9].copy_from_slice(b"MAINT"); + packet +} + +fn maintenance_clip_target() -> Arc { + Arc::new(crate::model::ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: shared::foundation::Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(arc_swap::ArcSwapOption::default()), + favourites: None, + processing_order: shared::model::ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }) +} + +#[tokio::test] +async fn new_ts_stream_for_input_in_maintenance_gets_the_clip_without_provider_slot() { + use axum::{http::HeaderMap, response::IntoResponse}; + use http_body_util::BodyExt; + + let app_state = create_test_provider_app_state(); + let mut cfg = (**app_state.app_config.config.load()).clone(); + cfg.custom_stream_response_enabled = true; + app_state.app_config.config.store(Arc::new(cfg)); + app_state.app_config.custom_stream_response.store(Some(Arc::new(crate::model::CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + maintenance: Some(crate::api::model::TransportStreamBuffer::new(maintenance_packet())), + panel_api_provisioning_hls_segments: Vec::new(), + }))); + let input = provider_state_input(&app_state); + app_state.maintenance.add(&input.name, None, 1).await.unwrap(); + // Unreachable: a provider connect attempt would fail instead of answering with the clip. + let stream_url = "http://127.0.0.1:1/live/user/pass/42.ts"; + + let response = stream_response( + &create_test_fingerprint("127.0.0.1:55201".parse().unwrap()), + &app_state, + "maintenance-session", + None, + create_test_live_channel(stream_url), + stream_url, + None, + &HeaderMap::default(), + &input, + &maintenance_clip_target(), + &viewer(), + shared::model::UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), axum::http::StatusCode::OK); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "no provider slot was taken"); + let body = tokio::time::timeout(std::time::Duration::from_secs(2), async { + let mut body = response.into_body(); + let frame = body.frame().await; + frame.and_then(Result::ok).and_then(|frame| frame.into_data().ok()).unwrap_or_default() + }) + .await + .unwrap_or_default(); + assert!(body.windows(5).any(|window| window == b"MAINT"), "the maintenance clip is served"); +} + +fn provider_state_input(app_state: &AppState) -> Arc { + Arc::clone(&app_state.app_config.sources.load().inputs[0]) +} diff --git a/backend/app/src/api/api_utils/tests/stalker.rs b/backend/app/src/api/api_utils/tests/stalker.rs index dde107e67..67fec2ae7 100644 --- a/backend/app/src/api/api_utils/tests/stalker.rs +++ b/backend/app/src/api/api_utils/tests/stalker.rs @@ -130,3 +130,36 @@ async fn resolve_streaming_strategy_rejects_stalker_url_after_forced_provider_fa app_state.active_provider.release_connection(&busy_addr); app_state.active_provider.release_connection(&fallback_addr); } + +#[tokio::test] +async fn initial_stalker_resolution_is_skipped_for_input_in_maintenance() { + let app_config = create_test_provider_app_config(); + let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; + let mut stalker_input = (*configured_input).clone(); + stalker_input.input_type = InputType::Stalker; + // Unreachable portal: a resolution attempt fails instead of answering. + stalker_input.url = "http://127.0.0.1:1".to_string(); + app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(stalker_input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(app_config)); + let input = Arc::clone(&app_state.app_config.sources.load().inputs[0]); + app_state.maintenance.add(&input.name, None, 1).await.unwrap(); + let mut viewer = crate::model::ProxyUserCredentials::default(); + viewer.username = "viewer".to_string(); + let empty: Arc = "".into(); + + let resolved = super::resolve_initial_stalker_playback_url( + &app_state, + &input, + &viewer, + 347, + XtreamCluster::Live, + PlaylistItemType::Live, + &empty, + ) + .await; + + // The portal is not contacted; the maintenance gate further down answers the request. + assert_eq!(resolved.ok().as_deref(), Some("")); +} diff --git a/backend/app/src/api/api_utils/tests/support.rs b/backend/app/src/api/api_utils/tests/support.rs index 5a262921b..387f4f5a3 100644 --- a/backend/app/src/api/api_utils/tests/support.rs +++ b/backend/app/src/api/api_utils/tests/support.rs @@ -450,6 +450,7 @@ pub(in crate::api::api_utils::tests) fn create_test_app_state_for_config(app_cfg recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + crate::api::model::create_test_maintenance(""), ), app_config: app_cfg, http_clients: Arc::default(), @@ -468,6 +469,7 @@ pub(in crate::api::api_utils::tests) fn create_test_app_state_for_config(app_cfg metadata_manager, auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + maintenance: crate::api::model::create_test_maintenance(""), }) } diff --git a/backend/app/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs index 98dddd02c..068f0721b 100644 --- a/backend/app/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -584,6 +584,7 @@ mod tests { )) .to_vec(), )), + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), } } @@ -717,6 +718,7 @@ mod tests { recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + crate::api::model::create_test_maintenance(""), ), app_config: app_cfg, http_clients: Arc::default(), @@ -735,6 +737,7 @@ mod tests { metadata_manager, auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + maintenance: crate::api::model::create_test_maintenance(""), }) } diff --git a/backend/app/src/api/endpoints/hls_api/manifest/publication.rs b/backend/app/src/api/endpoints/hls_api/manifest/publication.rs index 092fef3c3..e06c00a2a 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/publication.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/publication.rs @@ -44,6 +44,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_custom_video_type_for_failure_reas ConnectFailureReason::Preempted => CustomVideoStreamType::LowPriorityPreempted, ConnectFailureReason::Provisioning => CustomVideoStreamType::Provisioning, ConnectFailureReason::SessionExpired => CustomVideoStreamType::HlsSessionOrLeaseExpired, + ConnectFailureReason::Maintenance => CustomVideoStreamType::Maintenance, ConnectFailureReason::ProviderError | ConnectFailureReason::ProviderClosed | ConnectFailureReason::ChannelUnavailable => CustomVideoStreamType::ChannelUnavailable, diff --git a/backend/app/src/api/endpoints/hls_api/segment/entry.rs b/backend/app/src/api/endpoints/hls_api/segment/entry.rs index 60541c190..073eda608 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/entry.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/entry.rs @@ -11,8 +11,9 @@ use super::{ use crate::{ api::{ api_utils::{ - acquire_exact_provider_handle, connection_priority_for_kind, get_hls_playback_ttl_secs, - select_provider_stream_url, ExactProviderAcquire, HLS_MANIFEST_CAPACITY_WAIT, + acquire_exact_provider_handle, blocks_new_playback, connection_priority_for_kind, + get_hls_playback_ttl_secs, is_running_playback, select_provider_stream_url, ExactProviderAcquire, + HLS_MANIFEST_CAPACITY_WAIT, }, model::{AppState, CustomVideoStreamType, PlaybackLeaseRef, ProviderAllocation, UserSession}, }, @@ -87,6 +88,20 @@ pub(in crate::api) async fn handle_hls_stream_request( if app_state.active_users.is_user_blocked_for_stream(&user.username, VirtualId::new(virtual_id)).await { return axum::http::StatusCode::BAD_REQUEST.into_response(); } + // Only a fresh entry is gated. Variant refreshes, segments and entry re-polls of a running + // session (unwrapped media playlists) belong to a running playback. + if matches!(stage, HlsRequestStage::Entry) + && !is_running_playback(user_session) + && blocks_new_playback(app_state, input, user, false) + { + return hls_custom_video_manifest_response( + app_state, + user, + CustomVideoStreamType::Maintenance, + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + } let stream_ref = stream_context.stream_ref().to_string(); // Variant URLs were normalized before sealing or come from an upstream master; use them as is. diff --git a/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs index 8ececa9af..08b417e9c 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs @@ -1014,3 +1014,135 @@ async fn hls_proxy_manifest_invalid_token_starts_no_origin_work() { assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); assert!(app_state.hls.proxy.sessions().is_empty().await); } + +#[tokio::test] +async fn hls_entry_for_input_in_maintenance_opens_no_session() { + // Same setup as `hls_cache_entry_master_playlist_uses_cache_when_target_hls_share_enabled`, + // which answers 200 with one access lease when the input is not in maintenance. + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(true); + let original_hls_entry_path = super::super::build_virtual_hls_entry_path(&target, &input, &user, 1001); + app_state.maintenance.add(&input.name, None, 1).await.expect("maintenance"); + + let response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + None, + "http://origin.example.com/live/user/pass/1001.m3u8", + None, + test_hls_entry_stream_context(1001, "80510", Some(2_500_000)), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &original_hls_entry_path, + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + + // No maintenance clip is configured here, so the configured fallback status answers. + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(app_state.hls.proxy.access_leases().read().await.is_empty(), "no lease for a maintenance input"); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn hls_entry_refresh_of_a_running_session_is_not_gated() { + // Without media-playlist wrapping the player re-polls the entry URL; a known user session + // is a running playback and keeps playing through maintenance. + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(true); + let original_hls_entry_path = super::super::build_virtual_hls_entry_path(&target, &input, &user, 1001); + app_state.maintenance.add(&input.name, None, 1).await.expect("maintenance"); + let session = crate::api::model::UserSession { + token: "running-hls-session".to_string(), + virtual_id: 1001, + provider: Arc::clone(&input.name), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + Some(&session), + None, + "http://origin.example.com/live/user/pass/1001.m3u8", + None, + test_hls_entry_stream_context(1001, "80510", Some(2_500_000)), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &original_hls_entry_path, + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK, "a running session is not switched to the maintenance clip"); +} + +#[tokio::test] +async fn hls_entry_with_an_expired_session_is_gated() { + // A known but no longer counted session is not a running playback: it would be admitted + // anew, so maintenance answers it. + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(true); + let original_hls_entry_path = super::super::build_virtual_hls_entry_path(&target, &input, &user, 1001); + app_state.maintenance.add(&input.name, None, 1).await.expect("maintenance"); + let session = crate::api::model::UserSession { + token: "expired-hls-session".to_string(), + virtual_id: 1001, + provider: Arc::clone(&input.name), + lifecycle: crate::api::model::PlaybackLifecycle::Expired, + ..Default::default() + }; + + let response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + Some(&session), + None, + "http://origin.example.com/live/user/pass/1001.m3u8", + None, + test_hls_entry_stream_context(1001, "80510", Some(2_500_000)), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &original_hls_entry_path, + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(app_state.hls.proxy.access_leases().read().await.is_empty(), "no lease for a maintenance input"); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs b/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs index 46b7b94ce..4dffc61ac 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs @@ -193,6 +193,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn enable_provider_exhausted_custo user_account_expired: None, panel_api_provisioning: None, hls_session_or_lease_expired: None, + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), }))); } @@ -215,6 +216,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn enable_hls_provisioning_custom_ user_account_expired: None, panel_api_provisioning: None, hls_session_or_lease_expired: None, + maintenance: None, panel_api_provisioning_hls_segments: provisioning_segments, }))); } diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs index 44df8368f..80f0951cf 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs @@ -88,6 +88,34 @@ async fn hls_origin_account_io_lease_blocks_other_hls_sessions_for_same_account( wait_for_provider_connection_count(&app_state, 0).await; } +#[tokio::test] +async fn hls_origin_account_io_frees_its_slot_before_finish_returns() { + // The next origin I/O of the session (a manifest refresh right after a segment) or + // another session (a channel switch) must find the slot free at once. A deferred + // release makes it race for a slot the provider still counts, and an account at + // its connection limit answers that race with `Exhausted`. + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let other_session = create_bound_hls_test_session(&app_state, &input, "67890", "account-a", 1_000).await; + let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); + let other_binding = other_session.read().await.origin_account_binding.clone().expect("binding exists"); + let origin_io = test_hls_origin_io_context(&app_state); + + let first = begin_hls_origin_account_io(&origin_io, &session, &binding).await.expect("first io acquires"); + wait_for_provider_connection_count(&app_state, 1).await; + finish_hls_origin_account_io(&origin_io, &session, first, true).await; + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "released before finish returns"); + + let second = begin_hls_origin_account_io(&origin_io, &session, &binding).await.expect("next io acquires at once"); + finish_hls_origin_account_io(&origin_io, &session, second, true).await; + let other = begin_hls_origin_account_io(&origin_io, &other_session, &other_binding) + .await + .expect("another session acquires at once"); + finish_hls_origin_account_io(&origin_io, &other_session, other, true).await; + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); +} + #[tokio::test] async fn hls_account_overlap_selects_soft_candidate_but_not_hard_active() { let app_state = test_app_state(); diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs index ee59bdec1..b464e1b64 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs @@ -256,6 +256,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_app_state_with_hls_proxy_a recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + crate::api::model::create_test_maintenance(""), ), app_config, http_clients: Arc::new(tuliprox_core::model::HttpClients::new( @@ -283,6 +284,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_app_state_with_hls_proxy_a metadata_manager, auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + maintenance: crate::api::model::create_test_maintenance(""), }) } diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/http.rs b/backend/app/src/api/endpoints/hls_api/tests/support/http.rs index 5fd38a880..20ef221e0 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/http.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/http.rs @@ -26,6 +26,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn enable_channel_unavailable_cust user_account_expired: None, panel_api_provisioning: None, hls_session_or_lease_expired: None, + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), }))); } diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs index 6dbaca4d7..cf1b8a190 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs @@ -33,6 +33,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn enable_runtime_policy_custom_re user_account_expired: None, panel_api_provisioning: None, hls_session_or_lease_expired: None, + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), }))); } diff --git a/backend/app/src/api/endpoints/m3u_api.rs b/backend/app/src/api/endpoints/m3u_api.rs index 05fc50d6b..a6a7f2001 100644 --- a/backend/app/src/api/endpoints/m3u_api.rs +++ b/backend/app/src/api/endpoints/m3u_api.rs @@ -1,14 +1,15 @@ use crate::{ api::{ api_utils::{ - admission_failure_response, coalesce_byte_stream, create_m3u_catchup_session_key, + admission_failure_response, blocks_new_playback, coalesce_byte_stream, create_m3u_catchup_session_key, create_playback_session_fingerprint, create_session_fingerprint, force_provider_stream_response, - get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, + get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, is_running_playback, is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, local_stream_response, - redirect_response, reentry_suppressed_response, resolve_initial_stalker_playback_url, - resource_proxy_response, resource_redirect_or_proxy, separate_number_and_remainder, - should_allow_exhausted_shared_reconnect, stream_response_with_provider_handle, try_option_bad_request, - try_result_bad_request, try_result_not_found, try_unwrap_body, RedirectParams, + maintenance_rejection_response, redirect_response, reentry_suppressed_response, + resolve_initial_stalker_playback_url, resource_proxy_response, resource_redirect_or_proxy, + separate_number_and_remainder, should_allow_exhausted_shared_reconnect, + stream_response_with_provider_handle, try_option_bad_request, try_result_bad_request, try_result_not_found, + try_unwrap_body, RedirectParams, }, endpoints::{ hls_api::{ @@ -251,6 +252,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( pli.url = match resolve_initial_stalker_playback_url( app_state, &input, + &user, pli.get_provider_id().unwrap_or_default(), cluster, pli.item_type, @@ -380,6 +382,19 @@ pub(in crate::api) async fn m3u_api_stream_loaded( pli.url.clone() }; + if !is_running_playback(user_session.as_ref()) && blocks_new_playback(app_state, &input, &user, false) { + return maintenance_rejection_response( + app_state, + fingerprint, + &user, + stream_channel(), + &input, + req_headers, + extension == HLS_EXT, + ) + .await; + } + let (connection_admission, grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( &app_state.admission_ctx(), &user, diff --git a/backend/app/src/api/endpoints/maintenance_api.rs b/backend/app/src/api/endpoints/maintenance_api.rs new file mode 100644 index 000000000..e25e1ebb2 --- /dev/null +++ b/backend/app/src/api/endpoints/maintenance_api.rs @@ -0,0 +1,290 @@ +//! Operator control over input maintenance. +//! +//! Every call answers with the complete state, so a client never has to merge. + +use crate::api::{auth_middleware::permission_layer, model::AppState}; +use axum::{ + body::Bytes, + extract::{Path, State}, + http::StatusCode, + response::IntoResponse, + routing::get, + Json, Router, +}; +use log::error; +use shared::model::{ + normalize_maintenance_reason, MaintenanceRequestDto, MaintenanceStateDto, Permission, MAINTENANCE_WILDCARD, +}; +use std::sync::Arc; +use tuliprox_core::model::MaintenanceWrite; + +pub fn maintenance_api_register_with_permissions(app_state: &Arc) -> Router> { + let read_routes = Router::new() + .route("/maintenance", get(get_maintenance)) + .layer(permission_layer!(app_state, Permission::MaintenanceRead)); + let write_routes = Router::new() + .route("/maintenance/{input}", axum::routing::put(put_maintenance).delete(delete_maintenance)) + .layer(permission_layer!(app_state, Permission::MaintenanceWrite)); + Router::new().merge(read_routes).merge(write_routes) +} + +pub fn maintenance_api_register_unprotected() -> Router> { + Router::new() + .route("/maintenance", get(get_maintenance)) + .route("/maintenance/{input}", axum::routing::put(put_maintenance).delete(delete_maintenance)) +} + +fn current_state(app_state: &AppState) -> MaintenanceStateDto { + let sources = app_state.app_config.sources.load(); + MaintenanceStateDto { + entries: (*app_state.maintenance.snapshot()).clone(), + inputs: sources.inputs.iter().map(|input| input.name.to_string()).collect(), + } +} + +fn respond(app_state: &AppState, write: MaintenanceWrite) -> axum::response::Response { + if let Err(err) = app_state.publish_maintenance(write) { + error!("Could not persist maintenance state: {err}"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + Json(current_state(app_state)).into_response() +} + +async fn get_maintenance(State(app_state): State>) -> impl IntoResponse { + Json(current_state(&app_state)) +} + +/// The body is optional: a bare `PUT` puts the input into maintenance without a remark. +async fn put_maintenance( + State(app_state): State>, + Path(input): Path, + body: Bytes, +) -> axum::response::Response { + let request = if body.is_empty() { + MaintenanceRequestDto::default() + } else { + match serde_json::from_slice::(&body) { + Ok(request) => request, + Err(err) => return (StatusCode::BAD_REQUEST, err.to_string()).into_response(), + } + }; + let reason = match normalize_maintenance_reason(request.reason.as_deref()) { + Ok(reason) => reason, + Err(err) => return (StatusCode::BAD_REQUEST, err.to_string()).into_response(), + }; + let now = chrono::Utc::now().timestamp(); + let write = if input == MAINTENANCE_WILDCARD { + let sources = app_state.app_config.sources.load(); + let names: Vec<&str> = sources.inputs.iter().map(|input| input.name.as_ref()).collect(); + app_state.maintenance.add_all(&names, reason, now).await + } else if app_state.is_known_input_name(&input) { + app_state.maintenance.add(&input, reason, now).await + } else { + return StatusCode::NOT_FOUND.into_response(); + }; + respond(&app_state, write) +} + +async fn delete_maintenance( + State(app_state): State>, + Path(input): Path, +) -> axum::response::Response { + let write = if input == MAINTENANCE_WILDCARD { + app_state.maintenance.remove_all().await + } else if app_state.is_known_input_name(&input) { + app_state.maintenance.remove(&input).await + } else { + return StatusCode::NOT_FOUND.into_response(); + }; + respond(&app_state, write) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + api::model::create_test_app_state, + auth::create_jwt_web_user, + model::{Config, SourcesConfig}, + }; + use axum::{body::Body, http::Request}; + use http_body_util::BodyExt; + use shared::model::{ + ConfigInputDto, SourcesConfigDto, UserId, WebAuthConfigDto, WebUiConfigDto, MAINTENANCE_REASON_MAX_CHARS, + }; + use tempfile::TempDir; + use tower::ServiceExt; + + fn app_state_with(config: Config) -> Arc { + let state = create_test_app_state(config); + let dto = SourcesConfigDto { + inputs: ["provider_a", "provider_b"] + .iter() + .map(|name| ConfigInputDto { + name: (*name).to_string().into(), + url: "http://localhost/playlist.m3u".to_string(), + ..Default::default() + }) + .collect(), + ..Default::default() + }; + state.app_config.sources.store(Arc::new(SourcesConfig::try_from(&dto).unwrap())); + state + } + + fn app_state(dir: &TempDir) -> Arc { + app_state_with(Config { storage_dir: dir.path().to_string_lossy().to_string(), ..Config::default() }) + } + + async fn call(router: &Router, method: &str, uri: &str, body: Option<&str>) -> (StatusCode, Vec) { + let mut builder = Request::builder().method(method).uri(uri); + if body.is_some() { + builder = builder.header(axum::http::header::CONTENT_TYPE, "application/json"); + } + let request = builder.body(body.map_or_else(Body::empty, |b| Body::from(b.to_string()))).unwrap(); + let response = router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + (status, response.into_body().collect().await.unwrap().to_bytes().to_vec()) + } + + fn state_of(body: &[u8]) -> MaintenanceStateDto { serde_json::from_slice(body).unwrap() } + + #[tokio::test] + async fn get_lists_known_inputs_and_empty_entries() { + let dir = TempDir::new().unwrap(); + let router = maintenance_api_register_unprotected().with_state(app_state(&dir)); + let (status, body) = call(&router, "GET", "/maintenance", None).await; + assert_eq!(status, StatusCode::OK); + let state = state_of(&body); + assert!(state.entries.is_empty()); + assert_eq!(state.inputs, vec!["provider_a".to_string(), "provider_b".to_string()]); + } + + #[tokio::test] + async fn put_adds_with_trimmed_reason_and_delete_removes() { + let dir = TempDir::new().unwrap(); + let state = app_state(&dir); + let router = maintenance_api_register_unprotected().with_state(Arc::clone(&state)); + let (status, body) = call(&router, "PUT", "/maintenance/provider_a", Some(r#"{"reason":" down "}"#)).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(state_of(&body).entries["provider_a"].reason.as_deref(), Some("down")); + assert!(state.maintenance.is_in_maintenance("provider_a")); + + let (status, body) = call(&router, "DELETE", "/maintenance/provider_a", None).await; + assert_eq!(status, StatusCode::OK); + assert!(state_of(&body).entries.is_empty()); + } + + #[tokio::test] + async fn put_without_body_is_accepted() { + let dir = TempDir::new().unwrap(); + let router = maintenance_api_register_unprotected().with_state(app_state(&dir)); + let (status, body) = call(&router, "PUT", "/maintenance/provider_a", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(state_of(&body).entries["provider_a"].reason, None); + } + + #[tokio::test] + async fn invalid_json_body_is_bad_request() { + let dir = TempDir::new().unwrap(); + let router = maintenance_api_register_unprotected().with_state(app_state(&dir)); + let (status, _) = call(&router, "PUT", "/maintenance/provider_a", Some("{not json")).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn wildcard_adds_and_removes_all_known_inputs() { + let dir = TempDir::new().unwrap(); + let router = maintenance_api_register_unprotected().with_state(app_state(&dir)); + let (_, body) = call(&router, "PUT", "/maintenance/%2A", Some(r#"{"reason":"outage"}"#)).await; + assert_eq!(state_of(&body).entries.len(), 2); + let (_, body) = call(&router, "DELETE", "/maintenance/*", None).await; + assert!(state_of(&body).entries.is_empty()); + } + + #[tokio::test] + async fn unknown_input_is_not_found_and_changes_nothing() { + let dir = TempDir::new().unwrap(); + let state = app_state(&dir); + let router = maintenance_api_register_unprotected().with_state(Arc::clone(&state)); + let (status, _) = call(&router, "PUT", "/maintenance/nope", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (status, _) = call(&router, "DELETE", "/maintenance/nope", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(state.maintenance.snapshot().is_empty()); + } + + #[tokio::test] + async fn too_long_reason_is_bad_request() { + let dir = TempDir::new().unwrap(); + let router = maintenance_api_register_unprotected().with_state(app_state(&dir)); + let body = format!(r#"{{"reason":"{}"}}"#, "x".repeat(MAINTENANCE_REASON_MAX_CHARS + 1)); + let (status, _) = call(&router, "PUT", "/maintenance/provider_a", Some(&body)).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn delete_of_known_input_not_in_maintenance_is_ok() { + let dir = TempDir::new().unwrap(); + let router = maintenance_api_register_unprotected().with_state(app_state(&dir)); + let (status, _) = call(&router, "DELETE", "/maintenance/provider_b", None).await; + assert_eq!(status, StatusCode::OK); + } + + #[tokio::test] + async fn persist_failure_is_internal_error() { + let dir = TempDir::new().unwrap(); + let state = app_state(&dir); + let blocked = dir.path().join(crate::model::MAINTENANCE_FILE); + std::fs::create_dir(&blocked).unwrap(); + std::fs::write(blocked.join("blocker"), b"x").unwrap(); + let router = maintenance_api_register_unprotected().with_state(Arc::clone(&state)); + let (status, _) = call(&router, "PUT", "/maintenance/provider_a", None).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert!(!state.maintenance.is_in_maintenance("provider_a")); + } + + #[tokio::test] + async fn permissions_gate_read_and_write() -> Result<(), Box> { + let dir = TempDir::new()?; + let web_ui = WebUiConfigDto { + auth: Some(WebAuthConfigDto { + enabled: true, + issuer: "maintenance-test".to_owned(), + secret: "maintenance-test-secret".to_owned(), + ..Default::default() + }), + ..Default::default() + }; + let config = Config { + web_ui: Some((&web_ui).into()), + storage_dir: dir.path().to_string_lossy().to_string(), + ..Default::default() + }; + let Some(web_auth) = config.web_ui.as_ref().and_then(|web_ui| web_ui.auth.as_ref()) else { + return Err("missing web auth".into()); + }; + let reader = + create_jwt_web_user(web_auth, "reader", Permission::MaintenanceRead.into(), 0, UserId::from("web:reader"))?; + let system = + create_jwt_web_user(web_auth, "system", Permission::SystemRead.into(), 0, UserId::from("web:system"))?; + let state = app_state_with(config); + let router = maintenance_api_register_with_permissions(&state).with_state(Arc::clone(&state)); + let request = |method: &str, uri: &str, token: &str| { + Request::builder() + .method(method) + .uri(uri) + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + }; + + let response = router.clone().oneshot(request("GET", "/maintenance", &reader)?).await?; + assert_eq!(response.status(), StatusCode::OK); + let response = router.clone().oneshot(request("GET", "/maintenance", &system)?).await?; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = router.clone().oneshot(request("PUT", "/maintenance/provider_a", &reader)?).await?; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert!(!state.maintenance.is_in_maintenance("provider_a")); + Ok(()) + } +} diff --git a/backend/app/src/api/endpoints/mod.rs b/backend/app/src/api/endpoints/mod.rs index a9e1cb2dc..0348c28e4 100644 --- a/backend/app/src/api/endpoints/mod.rs +++ b/backend/app/src/api/endpoints/mod.rs @@ -8,6 +8,7 @@ mod hls_terminal_response; mod library_api; pub(in crate::api) mod log_ws_api; pub(in crate::api) mod m3u_api; +mod maintenance_api; pub(in crate::api) mod provider_resolve_api; mod rbac_api; pub(in crate::api) mod recording_api; diff --git a/backend/app/src/api/endpoints/provider_resolve_api.rs b/backend/app/src/api/endpoints/provider_resolve_api.rs index 6a959c7a9..af9558ad2 100644 --- a/backend/app/src/api/endpoints/provider_resolve_api.rs +++ b/backend/app/src/api/endpoints/provider_resolve_api.rs @@ -74,12 +74,15 @@ fn resolve_direct_provider_location(input: &ConfigInput, url: &str) -> Result, ) -> Result { - let redirect = user.proxy.is_redirect(item.item_type) || force_redirect; + // An input in maintenance is never redirected to its provider; the tuliprox stream URL + // answers with the maintenance clip. + let redirect = !input_in_maintenance && (user.proxy.is_redirect(item.item_type) || force_redirect); if redirect { return resolve_direct_provider_location(input, item.url); } @@ -219,6 +222,7 @@ async fn provider_resolve( match resolve_provider_playlist_item_location( user.as_ref(), target.is_force_redirect(item.item_type), + app_state.maintenance.is_in_maintenance(&input.name), output_kind, &server_info, &input, @@ -306,6 +310,7 @@ mod tests { let location = resolve_provider_playlist_item_location( &user(ProxyType::Redirect), false, + false, ProviderResolveOutputKind::Xtream, &server_info(), &input(), @@ -316,6 +321,30 @@ mod tests { assert_eq!(location, "http://provider.example.com/live/root/pass/81356.ts"); } + #[test] + fn provider_resolve_redirect_user_gets_tuliprox_location_for_input_in_maintenance() { + // A redirect would send the client to the provider; the tuliprox URL answers with the maintenance clip. + let item = ProviderResolveItem { + virtual_id: 81_356, + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + url: "provider://myprovider/live/root/pass/81356.ts", + }; + + let location = resolve_provider_playlist_item_location( + &user(ProxyType::Redirect), + false, + true, + ProviderResolveOutputKind::Xtream, + &server_info(), + &input(), + item, + ) + .unwrap(); + + assert!(location.starts_with("http://proxy.example.com/"), "{location}"); + } + #[test] fn provider_resolve_redirect_user_uses_current_provider_url() { let item = ProviderResolveItem { @@ -332,6 +361,7 @@ mod tests { let location = resolve_provider_playlist_item_location( &user(ProxyType::Redirect), false, + false, ProviderResolveOutputKind::Xtream, &server_info(), &input, @@ -354,6 +384,7 @@ mod tests { let location = resolve_provider_playlist_item_location( &user(ProxyType::Reverse(None)), false, + false, ProviderResolveOutputKind::Xtream, &server_info(), &input(), @@ -376,6 +407,7 @@ mod tests { let location = resolve_provider_playlist_item_location( &user(ProxyType::Reverse(None)), false, + false, ProviderResolveOutputKind::M3u, &server_info(), &input(), diff --git a/backend/app/src/api/endpoints/recording_api.rs b/backend/app/src/api/endpoints/recording_api.rs index c680ea3d1..e4047bda9 100644 --- a/backend/app/src/api/endpoints/recording_api.rs +++ b/backend/app/src/api/endpoints/recording_api.rs @@ -1207,6 +1207,8 @@ mod tests { fn release(&self, _handle: Option) -> BoxFuture<'_, ()> { Box::pin(async {}) } fn capacity_changed(&self) -> Arc { Arc::clone(&self.notify) } + + fn is_input_in_maintenance(&self, _input_name: &str) -> bool { false } } fn edit_claims(subject_id: Option, admin: bool) -> shared::model::Claims { diff --git a/backend/app/src/api/endpoints/v1_api.rs b/backend/app/src/api/endpoints/v1_api.rs index 944dfe8be..b19058625 100644 --- a/backend/app/src/api/endpoints/v1_api.rs +++ b/backend/app/src/api/endpoints/v1_api.rs @@ -199,6 +199,7 @@ pub fn v1_api_register( .merge(v1_api_playlist_register_with_permissions(axum::routing::Router::new(), app_state)) .merge(library_api_register(axum::routing::Router::new(), Some(app_state))) .merge(target_bouquet_api_register_with_permissions(app_state)) + .merge(super::maintenance_api::maintenance_api_register_with_permissions(app_state)) .merge(rbac_api_register(Arc::clone(app_state))) .merge(recording_availability_register(axum::routing::Router::new())) // `recording.enabled: false` turns the DVR off end to end: @@ -219,6 +220,7 @@ pub fn v1_api_register( .merge(v1_api_playlist_register_protected(axum::routing::Router::new())) .merge(library_api_register(axum::routing::Router::new(), None)) .merge(target_bouquet_api_register_unprotected()) + .merge(super::maintenance_api::maintenance_api_register_unprotected()) .merge(rbac_api_register_unprotected(Arc::clone(app_state))) .merge(recording_availability_register(axum::routing::Router::new())) .merge( diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_provider_resolution.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_provider_resolution.rs index dacd98980..5c62d24ef 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_provider_resolution.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_provider_resolution.rs @@ -469,6 +469,7 @@ async fn initial_stalker_playback_resolution_uses_the_requested_items_own_comman crate::api::api_utils::resolve_initial_stalker_playback_url( &app_state, &live_input, + &crate::model::ProxyUserCredentials::default(), provider_id, XtreamCluster::Live, live, @@ -501,6 +502,7 @@ async fn initial_stalker_playback_resolution_uses_the_requested_items_own_comman let passthrough = crate::api::api_utils::resolve_initial_stalker_playback_url( &app_state, &live_input, + &crate::model::ProxyUserCredentials::default(), 101, XtreamCluster::Live, live, diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs index 448dec1f9..09125f35a 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs @@ -152,6 +152,7 @@ pub(in crate::api::endpoints::v1_api_playlist::tests) fn test_app_state_with_man recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + crate::api::model::create_test_maintenance(""), ), app_config: app_cfg, http_clients: Arc::default(), @@ -170,5 +171,6 @@ pub(in crate::api::endpoints::v1_api_playlist::tests) fn test_app_state_with_man metadata_manager, auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests_with_sender(manual_update_sender), + maintenance: crate::api::model::create_test_maintenance(""), }) } diff --git a/backend/app/src/api/endpoints/websocket_api.rs b/backend/app/src/api/endpoints/websocket_api.rs index 569c8d49f..bd58467de 100644 --- a/backend/app/src/api/endpoints/websocket_api.rs +++ b/backend/app/src/api/endpoints/websocket_api.rs @@ -498,6 +498,10 @@ fn to_protocol_message(event: EventMessage) -> Option<(ProtocolMessage, &'static // enforces the session filter server-side, so this is a bare nudge: // the frontend re-fetches. EventMessage::RecordingRulesChanged => (ProtocolMessage::RecordingRulesChanged, "Recording rules changed"), + EventMessage::MaintenanceChanged(entries) => ( + ProtocolMessage::MaintenanceChangedResponse(unwrap_or_clone(entries)), + "Input maintenance changed", + ), // Handled by `handle_event_message`, not translatable here: // `RecordingChanged` needs a per-session snapshot re-fetch, and the @@ -936,6 +940,7 @@ mod tests { auth_audit(AuthAuditOutcome::SignInFailed), auth_audit(AuthAuditOutcome::SignInThrottled), auth_audit(AuthAuditOutcome::PermissionDenied), + EventMessage::MaintenanceChanged(Arc::default()), ]; assert_eq!(samples.len(), EventKind::ALL.len(), "add the new variant to this list"); samples @@ -1131,6 +1136,22 @@ mod tests { assert!(websocket_can_receive_runtime_events(&mem, &EventMessage::RecordingRulesChanged)); } + #[test] + fn maintenance_event_needs_maintenance_read() { + let event = EventMessage::MaintenanceChanged(Arc::default()); + let mut mem = ProtocolHandlerMemory { + permissions: Permission::MaintenanceRead.into(), + ..ProtocolHandlerMemory::default() + }; + mem.role = UserRole::User; + assert!(websocket_can_receive_runtime_events(&mem, &event)); + + let mut mem = + ProtocolHandlerMemory { permissions: Permission::SystemRead.into(), ..ProtocolHandlerMemory::default() }; + mem.role = UserRole::User; + assert!(!websocket_can_receive_runtime_events(&mem, &event)); + } + #[test] fn a_second_of_progress_reports_produces_one_snapshot() { // A running capture reports growth many times a second and each diff --git a/backend/app/src/api/endpoints/xtream_api/mod.rs b/backend/app/src/api/endpoints/xtream_api/mod.rs index 058bc4197..962cd4d2b 100644 --- a/backend/app/src/api/endpoints/xtream_api/mod.rs +++ b/backend/app/src/api/endpoints/xtream_api/mod.rs @@ -5,11 +5,12 @@ use crate::{ api::{ api_utils, api_utils::{ - admission_failure_response, coalesce_byte_stream, create_api_proxy_user, create_catchup_session_key, - create_m3u_catchup_session_key, create_playback_session_fingerprint, create_session_fingerprint, - empty_json_response_as_array, empty_json_response_as_object, force_provider_stream_response, - get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, internal_server_error, - is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, local_stream_response, + admission_failure_response, blocks_new_playback, coalesce_byte_stream, create_api_proxy_user, + create_catchup_session_key, create_m3u_catchup_session_key, create_playback_session_fingerprint, + create_session_fingerprint, empty_json_response_as_array, empty_json_response_as_object, + force_provider_stream_response, get_session_reservation_ttl_secs, get_user_target, + get_user_target_by_credentials, internal_server_error, is_running_playback, is_seekable_media_request, + is_session_based_playback, is_stream_share_enabled, local_stream_response, maintenance_rejection_response, redirect, redirect_response, reentry_suppressed_response, resolve_initial_stalker_playback_url, resource_proxy_response, resource_redirect_or_proxy, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, stream_response_with_provider_handle, @@ -443,6 +444,7 @@ async fn xtream_player_api_stream( pli.url = match resolve_initial_stalker_playback_url( app_state, &input, + &user, pli.provider_id, pli.xtream_cluster, item_type, @@ -577,6 +579,20 @@ async fn xtream_player_api_stream( pli.url.clone() }; + if !is_running_playback(user_session.as_ref()) && blocks_new_playback(app_state, &input, &user, false) { + let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); + return maintenance_rejection_response( + app_state, + fingerprint, + &user, + stream_channel, + &input, + req_headers, + playback_ext == HLS_EXT, + ) + .await; + } + let (connection_admission, grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( &app_state.admission_ctx(), &user, @@ -978,6 +994,7 @@ pub(in crate::api) async fn xtream_player_api_stream_with_resolved_target( pli.url = match resolve_initial_stalker_playback_url( app_state, &input, + &user, pli.provider_id, pli.xtream_cluster, resolution_item_type, @@ -1379,9 +1396,9 @@ pub async fn xtream_get_stream_info_response( return axum::Json(pli.to_info_document(&options)).into_response(); } - // handle upstream provider + // handle upstream provider; an input in maintenance answers from the fallback below if pli.provider_id > 0 { - if let Some(input) = input { + if let Some(input) = input.filter(|input| !app_state.maintenance.is_in_maintenance(&input.name)) { if let Some(info_url) = xtream::get_xtream_player_api_info_url(&input, cluster, pli.provider_id) { // redirect is only possible for live streams if user.proxy == ProxyType::Redirect && cluster == XtreamCluster::Live { @@ -1478,8 +1495,7 @@ async fn xtream_get_short_epg( } if pli.provider_id > 0 { - let input_name = &pli.input_name; - if let Some(input) = app_state.app_config.get_input_by_name(input_name) { + if let Some(input) = app_state.contactable_input(&pli.input_name) { if let Some(action_url) = xtream::get_xtream_player_api_action_url(&input, crate::model::XC_ACTION_GET_SHORT_EPG) { diff --git a/backend/app/src/api/endpoints/xtream_api/tests.rs b/backend/app/src/api/endpoints/xtream_api/tests.rs index d21240ec8..09e43d1ca 100644 --- a/backend/app/src/api/endpoints/xtream_api/tests.rs +++ b/backend/app/src/api/endpoints/xtream_api/tests.rs @@ -868,3 +868,153 @@ async fn xtream_token_stream_rejects_m3u_timeshift_with_bad_request() { assert_eq!(response.status(), axum::http::StatusCode::BAD_REQUEST); } + +/// Serves a recognisable clip for `maintenance`. +fn enable_maintenance_clip(app_state: &Arc) { + let mut cfg = (**app_state.app_config.config.load()).clone(); + cfg.custom_stream_response_enabled = true; + app_state.app_config.config.store(Arc::new(cfg)); + let mut packet = vec![0xff_u8; 188]; + packet[..4].copy_from_slice(&[0x47, 0x01, 0x00, 0x10]); + packet[4..9].copy_from_slice(b"MAINT"); + app_state.app_config.custom_stream_response.store(Some(Arc::new(crate::model::CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + maintenance: Some(crate::api::model::TransportStreamBuffer::new(packet)), + panel_api_provisioning_hls_segments: Vec::new(), + }))); +} + +/// Gives `user` (limited to one connection) a running stream on another channel. +async fn occupy_the_only_connection(app_state: &Arc, user: &ProxyUserCredentials) { + let busy_addr = "127.0.0.1:55401".parse().expect("addr"); + let busy = Fingerprint::new("fp-busy".to_string(), "127.0.0.1".to_string(), busy_addr); + app_state.connection_manager.add_connection(&busy_addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token: "busy-session", + virtual_id: 7, + provider: "provider-b", + stream_url: "http://other.example/live/7.ts", + addr: &busy_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let channel = shared::model::StreamChannel { + target_id: 1, + virtual_id: 7, + provider_id: 7, + input_name: "provider-b".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "Live".intern(), + title: "Busy".intern(), + url: "http://other.example/live/7.ts".into(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + }; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 401, + username: "busy", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &busy, + provider: "provider-b".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("busy-session"), + }) + .await; +} + +#[tokio::test] +async fn maintenance_answers_before_admission_for_an_exhausted_user() { + // An exhausted user asking for an input in maintenance gets the maintenance clip, and the + // request never reaches admission (no connection-denied event, no eviction). + let dir = tempdir().expect("temp dir"); + let config = Config { + storage_dir: dir.path().to_string_lossy().into_owned(), + user_access_control: true, + ..Config::default() + }; + let app_state = create_test_app_state(config); + enable_maintenance_clip(&app_state); + let target = Arc::new(short_epg_target()); + let mut item = short_epg_live_item(); + item.provider_id = 100; + item.input_name = "provider_a".intern(); + item.url = "http://127.0.0.1:1/live/user/pass/100.ts".intern(); + let input = ConfigInput { + id: 1, + name: "provider_a".intern(), + input_type: InputType::Xtream, + url: "http://127.0.0.1:1".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + max_connections: 5, + ..ConfigInput::default() + }; + cache_xtream_test_item(&app_state, &target, item, Some(input)).await; + app_state.maintenance.add("provider_a", None, 1).await.expect("maintenance"); + + let mut user = ProxyUserCredentials::default(); + user.username = "busy".to_string(); + user.password = "busy".to_string(); + user.max_connections = 1; + user.soft_connections = 0; + user.proxy = shared::model::ProxyType::Reverse(None); + user.output_clusters = ClusterFlags::all(); + occupy_the_only_connection(&app_state, &user).await; + + let addr = "127.0.0.1:55402".parse().expect("addr"); + let fingerprint = Fingerprint::new("fp-new".to_string(), "127.0.0.1".to_string(), addr); + let response = xtream_player_api_stream( + &fingerprint, + &HeaderMap::new(), + &app_state, + &UserApiRequest::default(), + ApiStreamRequest::from(ApiStreamContext::LiveAlt, "busy", "busy", "100.ts", ""), + Some((Arc::new(user), target)), + ) + .await + .into_response(); + + assert_eq!(response.status(), axum::http::StatusCode::OK, "the maintenance clip answers"); + let body = tokio::time::timeout(std::time::Duration::from_secs(2), async { + let mut body = response.into_body(); + let frame = http_body_util::BodyExt::frame(&mut body).await; + frame.and_then(Result::ok).and_then(|frame| frame.into_data().ok()).unwrap_or_default() + }) + .await + .unwrap_or_default(); + assert!(body.windows(5).any(|window| window == b"MAINT"), "the maintenance clip is served"); + let denied = app_state + .active_users + .events() + .stats() + .emitted() + .into_iter() + .find_map(|(kind, count)| (kind == shared::model::EventKind::ConnectionDenied).then_some(count)) + .unwrap_or(0); + assert_eq!(denied, 0, "the request never reached admission"); +} diff --git a/backend/app/src/api/main_api.rs b/backend/app/src/api/main_api.rs index 0a4fe4ca6..3a698103d 100644 --- a/backend/app/src/api/main_api.rs +++ b/backend/app/src/api/main_api.rs @@ -33,7 +33,10 @@ use crate::{ exec_config_watch, exec_interner_prune, exec_scheduler, exec_xtream_expiry_sync, spawn_ssdp_discover_task, }, }, - model::{AppConfig, Config, HdHomeRunFlags, Healthcheck, HttpClients, ProcessTargets, RateLimitConfig}, + model::{ + AppConfig, Config, HdHomeRunFlags, Healthcheck, HttpClients, MaintenanceRegistry, ProcessTargets, + RateLimitConfig, MAINTENANCE_FILE, + }, processing::processor::{exec_processing, next_playlist_update_run_order, ProcessingRun}, repository::{get_geoip_path, GeoIp}, utils::{exec_file_lock_prune, get_default_web_root_path}, @@ -189,6 +192,7 @@ fn spawn_metadata_trigger_update( .with_update_guard(update_guard.clone()) .with_disabled_headers(disabled_headers.clone()) .with_provider_manager(app_state_clone.active_provider.clone()) + .with_maintenance(Arc::clone(&app_state_clone.maintenance)) .with_metadata_manager(app_state_clone.metadata_manager.clone()) .with_pre_processed_inputs(pre_processed_inputs.clone()) .with_acquired_permit(lock), @@ -279,6 +283,14 @@ async fn ready( (status_code, axum::Json(ReadyResponse { status: status_label })) } +/// The persisted maintenance state, already pruned against the configured inputs. +pub(crate) async fn load_maintenance(app_config: &AppConfig) -> Arc { + let sources = app_config.sources.load(); + let known: Vec<&str> = sources.inputs.iter().map(|input| input.name.as_ref()).collect(); + let path = std::path::PathBuf::from(&app_config.config.load().storage_dir).join(MAINTENANCE_FILE); + Arc::new(MaintenanceRegistry::load(path, &known).await) +} + async fn create_shared_data( app_config: &Arc, forced_targets: &Arc, @@ -359,6 +371,8 @@ async fn create_shared_data( TuliproxError::Server(format!("Cannot load token revocations: {err}")) })?; + let maintenance = load_maintenance(app_config).await; + let app_state = AppState { app_config: Arc::clone(app_config), http_clients: Arc::new(HttpClients::new( @@ -377,6 +391,7 @@ async fn create_shared_data( recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + Arc::clone(&maintenance), ), active_provider, connection_manager, @@ -387,6 +402,7 @@ async fn create_shared_data( metadata_manager, auth: AuthState::new(token_revocations), playlist_updates: PlaylistUpdateControl::new(Arc::clone(forced_targets), manual_update_sender), + maintenance, }; Ok((app_state, manual_update_rx)) @@ -426,6 +442,7 @@ async fn run_manual_update_worker( .with_update_guard(app_state.playlist_updates.update_guard.clone()) .with_disabled_headers(app_state.get_disabled_headers()) .with_provider_manager(Arc::clone(&app_state.active_provider)) + .with_maintenance(Arc::clone(&app_state.maintenance)) .with_metadata_manager(Arc::clone(&app_state.metadata_manager)) .with_acquired_permit(permit), ) @@ -504,6 +521,7 @@ fn exec_update_on_boot(client: &reqwest::Client, app_state: &Arc, targ .with_update_guard(update_guard) .with_disabled_headers(disabled_headers) .with_provider_manager(provider_manager) + .with_maintenance(Arc::clone(&app_state_clone.maintenance)) .with_metadata_manager(metadata_manager), ) .await; @@ -1140,6 +1158,7 @@ mod tests { recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + crate::api::model::create_test_maintenance(""), ), app_config: app_cfg, http_clients: Arc::default(), @@ -1158,6 +1177,7 @@ mod tests { metadata_manager, auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + maintenance: crate::api::model::create_test_maintenance(""), }) } diff --git a/backend/app/src/api/model/app_state/mod.rs b/backend/app/src/api/model/app_state/mod.rs index 72335072f..d704af3f4 100644 --- a/backend/app/src/api/model/app_state/mod.rs +++ b/backend/app/src/api/model/app_state/mod.rs @@ -3,12 +3,15 @@ use crate::{ ActiveProviderManager, ActiveUserManager, ConnectionManager, EventManager, PlaylistStorage, PlaylistStorageState, RecordingQueue, SharedStreamManager, StalkerResolveCoordinator, }, - model::{AppConfig, GracePeriodOptions, HdHomeRunDeviceConfig, HttpClients, ReverseProxyDisabledHeaderConfig}, + model::{ + AppConfig, GracePeriodOptions, HdHomeRunDeviceConfig, HttpClients, MaintenanceRegistry, MaintenanceWrite, + ReverseProxyDisabledHeaderConfig, + }, repository::GeoIp, utils::LRUResourceCache, }; use arc_swap::{ArcSwap, ArcSwapOption}; -use shared::model::UserConnectionPermission; +use shared::model::{EventMessage, UserConnectionPermission}; use std::sync::{atomic::AtomicI8, Arc}; use tokio::sync::RwLock; use tuliprox_metadata::manager::MetadataUpdateManager; @@ -28,7 +31,7 @@ mod tests; mod view; #[cfg(test)] -pub use self::test_support::create_test_app_state; +pub use self::test_support::{create_test_app_state, create_test_maintenance}; pub use self::{ auth_state::*, cancel_tokens::*, hls_state::*, http_clients::*, playlist_update_control::*, reload::*, resource_cache::*, view::*, @@ -58,9 +61,32 @@ pub struct AppState { pub metadata_manager: Arc, pub auth: AuthState, pub playlist_updates: PlaylistUpdateControl, + /// Inputs taken out of service; read on every new stream request. + pub maintenance: Arc, } impl AppState { + /// The configured input, unless it is in maintenance: for code that would contact its provider. + pub fn contactable_input(&self, input_name: &Arc) -> Option> { + self.app_config.get_input_by_name(input_name).filter(|input| !self.maintenance.is_in_maintenance(&input.name)) + } + + pub fn is_known_input_name(&self, input_name: &str) -> bool { + self.app_config.sources.load().inputs.iter().any(|input| input.name.as_ref() == input_name) + } + + pub fn known_input_names(&self) -> Vec> { + self.app_config.sources.load().inputs.iter().map(|input| Arc::clone(&input.name)).collect() + } + + /// Emits the change event for a write that changed something. + pub fn publish_maintenance(&self, write: MaintenanceWrite) -> std::io::Result<()> { + if let Some(entries) = write? { + self.event_manager.send_event(EventMessage::MaintenanceChanged(entries)); + } + Ok(()) + } + pub async fn get_active_connections_for_user(&self, username: &str) -> u32 { self.active_users.user_connections(username).await } diff --git a/backend/app/src/api/model/app_state/reload.rs b/backend/app/src/api/model/app_state/reload.rs index a933374c4..400366f6b 100644 --- a/backend/app/src/api/model/app_state/reload.rs +++ b/backend/app/src/api/model/app_state/reload.rs @@ -80,6 +80,12 @@ pub async fn update_app_state_sources( }; app_state.playlist_updates.forced_targets.store(targets); let updates = app_state.set_sources(sources)?; + let known = app_state.known_input_names(); + let known: Vec<&str> = known.iter().map(AsRef::as_ref).collect(); + if let Err(err) = app_state.publish_maintenance(app_state.maintenance.retain_known(&known).await) { + // The reload itself succeeded; a stale entry only matches an input that no longer exists. + error!("Could not persist pruned maintenance state: {err}"); + } update_target_caches(app_state, updates.targets.as_ref()).await; restart_services(app_state, &updates); Ok(()) diff --git a/backend/app/src/api/model/app_state/test_support.rs b/backend/app/src/api/model/app_state/test_support.rs index f80f1bcbd..5dbf3e16a 100644 --- a/backend/app/src/api/model/app_state/test_support.rs +++ b/backend/app/src/api/model/app_state/test_support.rs @@ -63,6 +63,7 @@ pub fn create_test_app_state(config: Config) -> Arc { let tokens = CancelTokens::default(); let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); + let maintenance = super::create_test_maintenance(&app_config.config.load().storage_dir); Arc::new(AppState { app_config, http_clients: Arc::default(), @@ -75,6 +76,7 @@ pub fn create_test_app_state(config: Config) -> Arc { recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + Arc::clone(&maintenance), ), active_provider, connection_manager, @@ -85,5 +87,17 @@ pub fn create_test_app_state(config: Config) -> Arc { metadata_manager, auth: AuthState::for_tests(), playlist_updates: PlaylistUpdateControl::for_tests(), + maintenance, }) } + +/// A maintenance registry for tests: persisted under the configured storage dir, +/// or in memory only, so a test never writes into the working directory. +pub fn create_test_maintenance(storage_dir: &str) -> Arc { + if storage_dir.is_empty() { + return Arc::new(crate::model::MaintenanceRegistry::in_memory()); + } + Arc::new(crate::model::MaintenanceRegistry::new( + std::path::PathBuf::from(storage_dir).join(crate::model::MAINTENANCE_FILE), + )) +} diff --git a/backend/app/src/api/model/app_state/tests.rs b/backend/app/src/api/model/app_state/tests.rs index e76ba134e..14180c2f8 100644 --- a/backend/app/src/api/model/app_state/tests.rs +++ b/backend/app/src/api/model/app_state/tests.rs @@ -219,3 +219,60 @@ fn qos_aggregation_changed_detects_compaction_interval_changes() { assert!(qos_aggregation_changed(&old_config, &new_config)); } + +#[tokio::test] +async fn source_reload_prunes_removed_inputs_from_maintenance() { + use crate::{api::model::update_app_state_sources, model::SourcesConfig}; + use shared::model::{ConfigInputDto, EventMessage, SourcesConfigDto}; + use std::sync::Arc; + + let dir = tempfile::TempDir::new().unwrap(); + let config = Config { storage_dir: dir.path().to_string_lossy().to_string(), ..Config::default() }; + let app_state = super::create_test_app_state(config); + let sources = |names: &[&str]| { + let dto = SourcesConfigDto { + inputs: names + .iter() + .map(|name| ConfigInputDto { + name: (*name).to_string().into(), + url: "http://localhost/playlist.m3u".to_string(), + ..Default::default() + }) + .collect(), + ..Default::default() + }; + SourcesConfig::try_from(&dto).unwrap() + }; + app_state.app_config.sources.store(Arc::new(sources(&["provider_a", "provider_b"]))); + app_state.maintenance.add_all(&["provider_a", "provider_b"], None, 1).await.unwrap(); + let mut events = app_state.event_manager.get_event_channel(); + + update_app_state_sources(&app_state, sources(&["provider_a"]), None).await.unwrap(); + + assert!(app_state.maintenance.is_in_maintenance("provider_a")); + assert!(!app_state.maintenance.is_in_maintenance("provider_b")); + let pruned = std::iter::from_fn(|| events.try_recv().ok()) + .find_map(|event| match event { + EventMessage::MaintenanceChanged(entries) => Some(entries), + _ => None, + }) + .expect("pruning emits a change event"); + assert_eq!(pruned.len(), 1); +} + +#[tokio::test] +async fn contactable_input_hides_inputs_in_maintenance() { + use crate::model::{ConfigInput, SourcesConfig}; + use std::sync::Arc; + + let dir = tempfile::TempDir::new().unwrap(); + let config = Config { storage_dir: dir.path().to_string_lossy().to_string(), ..Config::default() }; + let app_state = super::create_test_app_state(config); + let input = Arc::new(ConfigInput { name: Arc::from("provider_a"), ..ConfigInput::default() }); + app_state.app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![input], ..SourcesConfig::default() })); + let name: Arc = Arc::from("provider_a"); + + assert!(app_state.contactable_input(&name).is_some()); + app_state.maintenance.add("provider_a", None, 1).await.unwrap(); + assert!(app_state.contactable_input(&name).is_none(), "an input in maintenance is not contacted"); +} diff --git a/backend/app/src/api/model/app_state/view.rs b/backend/app/src/api/model/app_state/view.rs index dc0f5f6cf..765063e87 100644 --- a/backend/app/src/api/model/app_state/view.rs +++ b/backend/app/src/api/model/app_state/view.rs @@ -51,6 +51,9 @@ impl AppState { /// event_manager` - or for a handle `AppState` keeps inside a container - /// `hls_proxy <- hls.proxy`. The context names the handle by the role it /// plays; the view still reads exactly the handle it names. +/// +/// A context that holds a handle as a trait object says so with `as` - +/// `maintenance as Arc`. macro_rules! view_field { ($state:ident, $field:ident) => { ::core::clone::Clone::clone(&$state.$field) @@ -63,12 +66,12 @@ macro_rules! view_field { macro_rules! app_state_views { ($( $(#[$meta:meta])* - $accessor:ident => $ctx:ty { $($field:ident $(<- $($source:ident).+)?),+ $(,)? } + $accessor:ident => $ctx:ty { $($field:ident $(<- $($source:ident).+)? $(as $cast:ty)?),+ $(,)? } )+) => { $( impl AppStateView for $ctx { fn from_app_state(state: &AppState) -> Self { - Self { $($field: view_field!(state, $field $(, $($source).+)?)),+ } + Self { $($field: view_field!(state, $field $(, $($source).+)?) $(as $cast)?),+ } } } @@ -110,5 +113,6 @@ app_state_views! { metadata_update_ctx => tuliprox_metadata::ctx::BoundMetadataUpdateCtx { app_config, active_provider, connection_manager, events <- event_manager, playlists, update_guard <- playlist_updates.update_guard, http_clients, + maintenance as std::sync::Arc, } } diff --git a/backend/app/src/api/model/hls_provisioning.rs b/backend/app/src/api/model/hls_provisioning.rs index 5fd782c84..f8448858c 100644 --- a/backend/app/src/api/model/hls_provisioning.rs +++ b/backend/app/src/api/model/hls_provisioning.rs @@ -341,6 +341,7 @@ fn custom_video_asset( CustomVideoStreamType::UserAccountExpired => response.user_account_expired.as_ref(), CustomVideoStreamType::Provisioning => response.panel_api_provisioning.as_ref(), CustomVideoStreamType::HlsSessionOrLeaseExpired => response.hls_session_or_lease_expired.as_ref(), + CustomVideoStreamType::Maintenance => response.maintenance.as_ref(), } } diff --git a/backend/app/src/api/model/recording_runtime.rs b/backend/app/src/api/model/recording_runtime.rs index bbc542b8b..2fdd264b8 100644 --- a/backend/app/src/api/model/recording_runtime.rs +++ b/backend/app/src/api/model/recording_runtime.rs @@ -9,7 +9,7 @@ use futures::future::BoxFuture; use std::sync::Arc; use tokio::sync::Notify; -use tuliprox_core::model::ProviderHandle; +use tuliprox_core::model::{MaintenanceGate, ProviderHandle}; use tuliprox_dvr::recording::recording_capacity::{ProviderCapacity, RecordingCapacityPort}; use tuliprox_session::{ActiveProviderManager, ConnectionManager}; @@ -17,11 +17,16 @@ use tuliprox_session::{ActiveProviderManager, ConnectionManager}; pub struct ProviderCapacityAdapter { active_provider: Arc, connection_manager: Arc, + maintenance: Arc, } impl ProviderCapacityAdapter { - pub fn new(active_provider: Arc, connection_manager: Arc) -> Arc { - Arc::new(Self { active_provider, connection_manager }) + pub fn new( + active_provider: Arc, + connection_manager: Arc, + maintenance: Arc, + ) -> Arc { + Arc::new(Self { active_provider, connection_manager, maintenance }) } } @@ -40,4 +45,6 @@ impl RecordingCapacityPort for ProviderCapacityAdapter { } fn capacity_changed(&self) -> Arc { self.connection_manager.capacity_notified() } + + fn is_input_in_maintenance(&self, input_name: &str) -> bool { self.maintenance.is_in_maintenance(input_name) } } diff --git a/backend/app/src/api/model/streams/active_client_stream/state.rs b/backend/app/src/api/model/streams/active_client_stream/state.rs index 7f7f09216..ae75c4408 100644 --- a/backend/app/src/api/model/streams/active_client_stream/state.rs +++ b/backend/app/src/api/model/streams/active_client_stream/state.rs @@ -161,7 +161,9 @@ impl ActiveClientStreamState { CustomVideoStreamType::ProviderConnectionsExhausted => Some(StreamMode::ProviderExhausted), CustomVideoStreamType::LowPriorityPreempted => Some(StreamMode::LowPriorityPreempted), CustomVideoStreamType::Provisioning => Some(StreamMode::Provisioning), - CustomVideoStreamType::UserAccountExpired | CustomVideoStreamType::HlsSessionOrLeaseExpired => None, + CustomVideoStreamType::UserAccountExpired + | CustomVideoStreamType::HlsSessionOrLeaseExpired + | CustomVideoStreamType::Maintenance => None, } } diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs b/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs index e1d9bc9c6..5a71afef0 100644 --- a/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs +++ b/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs @@ -70,6 +70,7 @@ pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_a recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + crate::api::model::create_test_maintenance(""), ), app_config: Arc::new(app_cfg), http_clients: Arc::default(), @@ -88,6 +89,7 @@ pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_a metadata_manager, auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + maintenance: crate::api::model::create_test_maintenance(""), }) } diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/support.rs b/backend/app/src/api/model/streams/active_client_stream/tests/support.rs index 8135f1764..b85a40a93 100644 --- a/backend/app/src/api/model/streams/active_client_stream/tests/support.rs +++ b/backend/app/src/api/model/streams/active_client_stream/tests/support.rs @@ -102,6 +102,7 @@ pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_a recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( Arc::clone(&active_provider), Arc::clone(&connection_manager), + crate::api::model::create_test_maintenance(""), ), app_config: app_cfg, http_clients: Arc::default(), @@ -120,6 +121,7 @@ pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_a metadata_manager, auth: crate::api::model::AuthState::for_tests(), playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + maintenance: crate::api::model::create_test_maintenance(""), }) } diff --git a/backend/app/src/api/model/streams/provider_stream.rs b/backend/app/src/api/model/streams/provider_stream.rs index a5be0eab8..523705ff5 100644 --- a/backend/app/src/api/model/streams/provider_stream.rs +++ b/backend/app/src/api/model/streams/provider_stream.rs @@ -177,6 +177,13 @@ ok_custom_stream_factory!( "Streaming response hls session or lease expired" ); +ok_custom_stream_factory!( + create_maintenance_stream, + maintenance, + CustomVideoStreamType::Maintenance, + "Streaming response input maintenance" +); + pub fn create_panel_api_provisioning_stream_with_stop( cfg: &AppConfig, headers: &[(String, String)], @@ -216,6 +223,7 @@ pub fn create_custom_video_stream_response( CustomVideoStreamType::UserAccountExpired => create_user_account_expired_stream(config, &[]), CustomVideoStreamType::Provisioning => create_panel_api_provisioning_stream(config, &[]), CustomVideoStreamType::HlsSessionOrLeaseExpired => create_hls_session_or_lease_expired_stream(config, &[]), + CustomVideoStreamType::Maintenance => create_maintenance_stream(config, &[]), } { let mut builder = axum::response::Response::builder().status(status_code); for (key, value) in headers { @@ -327,6 +335,7 @@ mod tests { user_account_expired: None, panel_api_provisioning: None, hls_session_or_lease_expired: None, + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), }))); app_cfg @@ -368,6 +377,31 @@ mod tests { assert!(matches!(stream_type, Some(CustomVideoStreamType::ChannelUnavailable))); } + #[tokio::test] + async fn maintenance_factory_serves_its_own_clip() { + use super::create_maintenance_stream; + use crate::api::model::TransportStreamBuffer; + + let mut ts_packet = vec![0_u8; 188]; + ts_packet[0] = 0x47; + let app_cfg = create_test_app_config_with_channel_unavailable(); + let mut current = (*app_cfg.custom_stream_response.load_full().expect("custom response set")).clone(); + current.maintenance = Some(TransportStreamBuffer::new(ts_packet)); + app_cfg.custom_stream_response.store(Some(Arc::new(current))); + + let (_stream, info) = create_maintenance_stream(&app_cfg, &[]); + let (_headers, status, _url, stream_type) = info.expect("input maintenance clip should exist"); + assert_eq!(status, StatusCode::OK); + assert!(matches!(stream_type, Some(CustomVideoStreamType::Maintenance))); + } + + #[tokio::test] + async fn maintenance_without_clip_has_no_stream() { + let app_cfg = create_test_app_config_with_channel_unavailable(); + let (stream, info) = super::create_maintenance_stream(&app_cfg, &[]); + assert!(stream.is_none() && info.is_none()); + } + /// The `ok_custom_stream_factory!` macro generates a public factory for /// each (field, stream type, description) tuple. Pick one representative /// factory — `create_user_connections_exhausted_stream` — and verify the diff --git a/backend/app/src/api/panel_api/alias_pool.rs b/backend/app/src/api/panel_api/alias_pool.rs index cbdcffced..e85a0239c 100644 --- a/backend/app/src/api/panel_api/alias_pool.rs +++ b/backend/app/src/api/panel_api/alias_pool.rs @@ -512,6 +512,9 @@ pub(crate) async fn sync_panel_api_exp_dates(app_state: &Arc) { let sources = app_state.app_config.sources.load(); for input in &sources.inputs { + if app_state.maintenance.is_in_maintenance(&input.name) { + continue; + } if sync_panel_api_for_input_on_boot(app_state, input, sources_path.as_path()).await { any_change = true; } @@ -543,6 +546,9 @@ pub(crate) async fn sync_panel_api_alias_pool_for_target(app_state: &Arc Option { let title = "Recording rules changed".to_string(); Some(NotificationEvent::new(id, title.clone(), title)) } + EventMessage::MaintenanceChanged(entries) => { + let title = if entries.is_empty() { + "No input in maintenance".to_string() + } else { + let names: Vec<&str> = entries.keys().map(String::as_str).collect(); + format!("Inputs in maintenance: {}", names.join(", ")) + }; + Some(NotificationEvent::new(id, title.clone(), title)) + } // Events that already have a `MessageContent` shape reuse its // builders: `from_content` is what built their title, body and @@ -561,6 +571,7 @@ mod tests { EventMessage::RecordingChanged, EventMessage::RecordingProgress, EventMessage::RecordingRulesChanged, + EventMessage::MaintenanceChanged(std::sync::Arc::default()), EventMessage::InputMetadataUpdatesStarted("a".into()), EventMessage::InputMetadataUpdatesFailed(MetadataUpdateFailure::new("a".into(), 1, false, None)), EventMessage::InputMetadataUpdatesCompleted("a".into()), @@ -705,6 +716,7 @@ mod tests { EventMessage::ActiveProvider("provider-a".into(), 3), EventMessage::RecordingChanged, EventMessage::RecordingRulesChanged, + EventMessage::MaintenanceChanged(std::sync::Arc::default()), ]; for message in messages { let event = to_notification(&message).expect("mapped"); @@ -726,6 +738,7 @@ mod tests { EventMessage::ActiveProvider("a".into(), 1), EventMessage::RecordingChanged, EventMessage::RecordingRulesChanged, + EventMessage::MaintenanceChanged(std::sync::Arc::default()), ]; for message in messages { let event = to_notification(&message).expect("mapped"); diff --git a/backend/app/src/api/tasks/scheduler.rs b/backend/app/src/api/tasks/scheduler.rs index f864ae25c..c25c57979 100644 --- a/backend/app/src/api/tasks/scheduler.rs +++ b/backend/app/src/api/tasks/scheduler.rs @@ -205,6 +205,7 @@ async fn run_playlist_update_inner( .with_update_guard(app_state.playlist_updates.update_guard.clone()) .with_disabled_headers(app_state.get_disabled_headers()) .with_provider_manager(Arc::clone(&app_state.active_provider)) + .with_maintenance(Arc::clone(&app_state.maintenance)) .with_metadata_manager(Arc::clone(&app_state.metadata_manager)) .with_acquired_permit(permit), ) diff --git a/backend/app/src/api/tasks/xtream_expiry.rs b/backend/app/src/api/tasks/xtream_expiry.rs index 9a73cbedb..18a822203 100644 --- a/backend/app/src/api/tasks/xtream_expiry.rs +++ b/backend/app/src/api/tasks/xtream_expiry.rs @@ -64,8 +64,9 @@ pub fn exec_xtream_expiry_sync(app_state: &Arc, cancel: &CancellationT let mut state_dirty = false; loop { let now = Utc::now().timestamp(); - let accounts = collect_accounts(&app_state); + let mut accounts = collect_accounts(&app_state); state_dirty |= prune_state(&mut state, &accounts, now); + retain_queryable(&app_state, &mut accounts); fetch_due_accounts( &app_state, &accounts, @@ -255,6 +256,12 @@ fn prune_state(state: &mut ExpiryState, accounts: &[Account], now: i64) -> bool + state.cooldown_until.len() } +/// An input in maintenance keeps its throttle state (pruning sees every account) but is +/// neither queried nor persisted; its accounts are picked up again afterwards. +fn retain_queryable(app_state: &AppState, accounts: &mut Vec) { + accounts.retain(|account| !app_state.maintenance.is_in_maintenance(&account.input_name)); +} + fn collect_accounts(app_state: &AppState) -> Vec { let sources = app_state.app_config.sources.load(); sources @@ -514,6 +521,22 @@ mod tests { } } + #[tokio::test] + async fn accounts_of_inputs_in_maintenance_are_not_queried() -> Result<(), Box> { + let (_dir, app, _path) = super::super::provider_account_state::tests::fixture(false).await?; + let mut accounts = super::collect_accounts(&app); + super::retain_queryable(&app, &mut accounts); + assert!(!accounts.is_empty(), "the fixture input is queried normally"); + + app.maintenance.add("root", None, 1).await?; + + let mut accounts = super::collect_accounts(&app); + assert!(!accounts.is_empty(), "pruning still sees accounts of an input in maintenance"); + super::retain_queryable(&app, &mut accounts); + assert!(accounts.is_empty(), "an input in maintenance is not contacted"); + Ok(()) + } + #[tokio::test] async fn restored_pending_csv_expiry_updates_runtime_without_reload() -> Result<(), Box> { let (_dir, app, path) = super::super::provider_account_state::tests::fixture(true).await?; diff --git a/backend/app/src/main.rs b/backend/app/src/main.rs index c6457735c..7535392d7 100644 --- a/backend/app/src/main.rs +++ b/backend/app/src/main.rs @@ -548,8 +548,14 @@ async fn start_in_cli_mode(cfg: Arc, targets: Arc) { error!("Failed to build client {err}"); reqwest::Client::new() }); - // In CLI mode, we don't start background managers for events or providers - exec_processing(ProcessingRun::new(client, cfg, targets, shared::model::NoopSink)).await; + // In CLI mode, we don't start background managers for events or providers. Maintenance + // still applies: an input taken out of service is not fetched by a one-shot run either. + // Read-only: a one-shot run may share the storage directory with a running server. + let storage_dir = arc_swap::ArcSwapAny::load(cfg.config.as_ref()).storage_dir.clone(); + let maintenance_path = std::path::PathBuf::from(storage_dir).join(tuliprox_core::model::MAINTENANCE_FILE); + let maintenance = Arc::new(tuliprox_core::model::MaintenanceRegistry::load_read_only(maintenance_path).await); + exec_processing(ProcessingRun::new(client, cfg, targets, shared::model::NoopSink).with_maintenance(maintenance)) + .await; } async fn start_in_server_mode(cfg: Arc, targets: Arc) { diff --git a/backend/auth/src/recording_auth.rs b/backend/auth/src/recording_auth.rs index c158c151a..02e3bc7ae 100644 --- a/backend/auth/src/recording_auth.rs +++ b/backend/auth/src/recording_auth.rs @@ -110,6 +110,8 @@ fn permission_name(p: Permission) -> &'static str { Permission::RecordingCreate => "recording.create", Permission::RecordingManage => "recording.manage", Permission::RecordingDelete => "recording.delete", + Permission::MaintenanceRead => "maintenance.read", + Permission::MaintenanceWrite => "maintenance.write", } } @@ -443,6 +445,7 @@ mod tests { measured_bytes: 0, completed_at: None, notification_markers: Vec::new(), + provider_started: false, group: None, series_name: None, deleting_previous_state: None, diff --git a/backend/core/src/model/config/app.rs b/backend/core/src/model/config/app.rs index 1e86ce0c1..3800ac31e 100644 --- a/backend/core/src/model/config/app.rs +++ b/backend/core/src/model/config/app.rs @@ -11,7 +11,7 @@ use log::{error, warn}; use rand::Rng; use shared::{ defaults::{ - CHANNEL_UNAVAILABLE, HLS_SESSION_OR_LEASE_EXPIRED, LOW_PRIORITY_PREEMPTED, PANEL_API_PROVISIONING, + CHANNEL_UNAVAILABLE, HLS_SESSION_OR_LEASE_EXPIRED, LOW_PRIORITY_PREEMPTED, MAINTENANCE, PANEL_API_PROVISIONING, PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT, PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX, PROVIDER_CONNECTIONS_EXHAUSTED, USER_ACCOUNT_EXPIRED, USER_CONNECTIONS_EXHAUSTED, }, @@ -509,6 +509,7 @@ impl AppConfig { let user_account_expired = load_and_set_file(&path.join(USER_ACCOUNT_EXPIRED)); let panel_api_provisioning = load_and_set_file(&path.join(PANEL_API_PROVISIONING)); let hls_session_or_lease_expired = load_and_set_file(&path.join(HLS_SESSION_OR_LEASE_EXPIRED)); + let maintenance = load_and_set_file(&path.join(MAINTENANCE)); let panel_api_provisioning_hls_segments = (0..PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT) .filter_map(|index| { let filename = format!("{PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX}{index:03}.ts"); @@ -523,6 +524,7 @@ impl AppConfig { user_account_expired, panel_api_provisioning, hls_session_or_lease_expired, + maintenance, panel_api_provisioning_hls_segments, }))); } diff --git a/backend/core/src/model/config/stream.rs b/backend/core/src/model/config/stream.rs index 8e58d6bc2..1c5a8b769 100644 --- a/backend/core/src/model/config/stream.rs +++ b/backend/core/src/model/config/stream.rs @@ -207,5 +207,6 @@ pub struct CustomStreamResponse { pub user_account_expired: Option, pub panel_api_provisioning: Option, pub hls_session_or_lease_expired: Option, + pub maintenance: Option, // input taken out of service by an operator pub panel_api_provisioning_hls_segments: Vec, } diff --git a/backend/core/src/model/maintenance.rs b/backend/core/src/model/maintenance.rs new file mode 100644 index 000000000..9fe7691bc --- /dev/null +++ b/backend/core/src/model/maintenance.rs @@ -0,0 +1,390 @@ +//! Inputs an operator has taken out of service. +//! +//! Read on every new stream request and every background provider job, written +//! a few times a day. Readers load one atomic pointer; writers serialize, persist +//! and only then publish, so a failed write never changes what readers see. + +use arc_swap::ArcSwapOption; +use log::{error, warn}; +use shared::model::{InputMaintenance, MaintenanceEntries}; +use std::{path::PathBuf, sync::Arc}; + +pub const MAINTENANCE_FILE: &str = "maintenance.json"; + +/// The read side, for crates that must not depend on the registry itself. +pub trait MaintenanceGate: Send + Sync { + fn is_in_maintenance(&self, input_name: &str) -> bool; +} + +/// `Ok(Some(state))` when the call changed something, `Ok(None)` for a no-op. +pub type MaintenanceWrite = std::io::Result>>; + +const REFUSE_UNREADABLE: &str = "maintenance state could not be read; it is not overwritten"; +const REFUSE_READ_ONLY: &str = "maintenance state was loaded read-only"; + +pub struct MaintenanceRegistry { + /// `None` while no input is in maintenance, which keeps the common check to one load. + entries: ArcSwapOption, + write_lock: tokio::sync::Mutex<()>, + /// Where the state is persisted; `None` keeps it in memory only. + path: Option, + /// Why writes are refused, if they are. + refuse_writes: Option<&'static str>, +} + +impl MaintenanceRegistry { + #[must_use] + pub fn new(path: PathBuf) -> Self { Self::with(Some(path), MaintenanceEntries::new(), None) } + + /// A registry that never touches the file system. + #[must_use] + pub fn in_memory() -> Self { Self::with(None, MaintenanceEntries::new(), None) } + + fn with(path: Option, entries: MaintenanceEntries, refuse_writes: Option<&'static str>) -> Self { + Self { + entries: ArcSwapOption::new(non_empty(entries)), + write_lock: tokio::sync::Mutex::new(()), + path, + refuse_writes, + } + } + + /// Loads the persisted state and drops inputs that no longer exist. + /// + /// A corrupt file must not stop the server: it is moved aside and the + /// registry starts empty, which the dashboard card makes visible. A file that + /// cannot be read at all stays where it is and is never overwritten. + pub async fn load(path: PathBuf, known: &[&str]) -> Self { + let (entries, refuse_writes) = match tokio::fs::read(&path).await { + Ok(bytes) => match serde_json::from_slice::(&bytes) { + Ok(entries) => (entries, None), + Err(err) => { + error!("Unreadable maintenance state {}: {err}", path.display()); + let corrupt = path.with_extension("json.corrupt"); + if let Err(err) = tokio::fs::rename(&path, &corrupt).await { + error!("Could not move {} aside: {err}", path.display()); + } + (MaintenanceEntries::new(), None) + } + }, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => (MaintenanceEntries::new(), None), + Err(err) => { + error!("Could not read maintenance state {}: {err}; changes are refused", path.display()); + (MaintenanceEntries::new(), Some(REFUSE_UNREADABLE)) + } + }; + let registry = Self::with(Some(path), entries, refuse_writes); + if registry.refuse_writes.is_none() { + if let Err(err) = registry.retain_known(known).await { + warn!("Could not persist pruned maintenance state: {err}"); + } + } + registry + } + + /// Loads the persisted state without ever changing the file, for a run that may + /// share the storage directory with a running server. + pub async fn load_read_only(path: PathBuf) -> Self { + let entries = match tokio::fs::read(&path).await { + Ok(bytes) => serde_json::from_slice::(&bytes).unwrap_or_else(|err| { + warn!("Ignoring unreadable maintenance state {}: {err}", path.display()); + MaintenanceEntries::new() + }), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => MaintenanceEntries::new(), + Err(err) => { + warn!("Could not read maintenance state {}: {err}", path.display()); + MaintenanceEntries::new() + } + }; + Self::with(Some(path), entries, Some(REFUSE_READ_ONLY)) + } + + #[inline] + pub fn is_in_maintenance(&self, input_name: &str) -> bool { + self.entries.load().as_ref().is_some_and(|entries| entries.contains_key(input_name)) + } + + /// The current state; an empty map when no input is in maintenance. + pub fn snapshot(&self) -> Arc { self.entries.load_full().unwrap_or_default() } + + pub async fn add(&self, name: &str, reason: Option, now: i64) -> MaintenanceWrite { + self.mutate(|entries| upsert(entries, name, reason, now)).await + } + + pub async fn add_all(&self, names: &[&str], reason: Option, now: i64) -> MaintenanceWrite { + self.mutate(|entries| { + let mut changed = false; + for name in names { + if !entries.contains_key(*name) { + entries.insert((*name).to_string(), InputMaintenance { since: now, reason: reason.clone() }); + changed = true; + } + } + changed + }) + .await + } + + pub async fn remove(&self, name: &str) -> MaintenanceWrite { + if !self.is_in_maintenance(name) { + return Ok(None); + } + self.mutate(|entries| entries.remove(name).is_some()).await + } + + pub async fn remove_all(&self) -> MaintenanceWrite { + if self.entries.load().is_none() { + return Ok(None); + } + self.mutate(|entries| { + let changed = !entries.is_empty(); + entries.clear(); + changed + }) + .await + } + + /// Drops entries whose input is gone. Checks the snapshot first, so a reload + /// that keeps every input takes no lock and does no IO. + pub async fn retain_known(&self, known: &[&str]) -> MaintenanceWrite { + let has_orphans = self + .entries + .load() + .as_ref() + .is_some_and(|entries| entries.keys().any(|name| !known.contains(&name.as_str()))); + if !has_orphans { + return Ok(None); + } + self.mutate(|entries| { + let before = entries.len(); + entries.retain(|name, _| known.contains(&name.as_str())); + entries.len() != before + }) + .await + } + + async fn mutate(&self, apply: impl FnOnce(&mut MaintenanceEntries) -> bool) -> MaintenanceWrite { + if let Some(reason) = self.refuse_writes { + return Err(std::io::Error::other(reason)); + } + let _guard = self.write_lock.lock().await; + let mut entries = self.entries.load().as_deref().cloned().unwrap_or_default(); + if !apply(&mut entries) { + return Ok(None); + } + if let Some(path) = &self.path { + let encoded = serde_json::to_vec(&entries).map_err(std::io::Error::other)?; + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + crate::utils::write_json_atomic(path, &encoded).await?; + } + let published = Arc::new(entries); + self.entries.store((!published.is_empty()).then(|| Arc::clone(&published))); + Ok(Some(published)) + } +} + +impl MaintenanceGate for MaintenanceRegistry { + #[inline] + fn is_in_maintenance(&self, input_name: &str) -> bool { Self::is_in_maintenance(self, input_name) } +} + +fn non_empty(entries: MaintenanceEntries) -> Option> { + (!entries.is_empty()).then(|| Arc::new(entries)) +} + +fn upsert(entries: &mut MaintenanceEntries, name: &str, reason: Option, now: i64) -> bool { + match entries.get_mut(name) { + Some(existing) if existing.reason == reason => false, + Some(existing) => { + existing.reason = reason; + true + } + None => { + entries.insert(name.to_string(), InputMaintenance { since: now, reason }); + true + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn registry_in(dir: &TempDir) -> MaintenanceRegistry { MaintenanceRegistry::new(dir.path().join(MAINTENANCE_FILE)) } + + #[tokio::test] + async fn empty_registry_reports_nothing_in_maintenance() { + let dir = TempDir::new().unwrap(); + let registry = registry_in(&dir); + assert!(!registry.is_in_maintenance("provider_a")); + assert!(registry.snapshot().is_empty()); + } + + #[tokio::test] + async fn add_persists_and_publishes() { + let dir = TempDir::new().unwrap(); + let registry = registry_in(&dir); + let changed = registry.add("provider_a", Some("down".into()), 100).await.unwrap(); + assert!(changed.is_some()); + assert!(registry.is_in_maintenance("provider_a")); + assert!(!registry.is_in_maintenance("provider_b")); + + let reloaded = MaintenanceRegistry::load(dir.path().join(MAINTENANCE_FILE), &["provider_a"]).await; + let entry = reloaded.snapshot().get("provider_a").cloned().unwrap(); + assert_eq!(entry, InputMaintenance { since: 100, reason: Some("down".into()) }); + } + + #[tokio::test] + async fn re_add_updates_reason_and_keeps_since() { + let dir = TempDir::new().unwrap(); + let registry = registry_in(&dir); + registry.add("provider_a", Some("first".into()), 100).await.unwrap(); + registry.add("provider_a", Some("second".into()), 200).await.unwrap(); + let entry = registry.snapshot().get("provider_a").cloned().unwrap(); + assert_eq!(entry, InputMaintenance { since: 100, reason: Some("second".into()) }); + } + + #[tokio::test] + async fn re_add_with_same_reason_is_a_no_op() { + let dir = TempDir::new().unwrap(); + let registry = registry_in(&dir); + registry.add("provider_a", None, 100).await.unwrap(); + assert!(registry.add("provider_a", None, 200).await.unwrap().is_none()); + } + + #[tokio::test] + async fn add_all_keeps_existing_entries() { + let dir = TempDir::new().unwrap(); + let registry = registry_in(&dir); + registry.add("provider_a", Some("old".into()), 100).await.unwrap(); + registry.add_all(&["provider_a", "provider_b"], Some("bulk".into()), 200).await.unwrap(); + let snapshot = registry.snapshot(); + assert_eq!(snapshot["provider_a"], InputMaintenance { since: 100, reason: Some("old".into()) }); + assert_eq!(snapshot["provider_b"], InputMaintenance { since: 200, reason: Some("bulk".into()) }); + } + + #[tokio::test] + async fn remove_and_remove_all_normalize_to_empty() { + let dir = TempDir::new().unwrap(); + let registry = registry_in(&dir); + registry.add_all(&["provider_a", "provider_b"], None, 100).await.unwrap(); + registry.remove("provider_a").await.unwrap(); + assert!(!registry.is_in_maintenance("provider_a")); + assert!(registry.remove("provider_a").await.unwrap().is_none(), "second remove is a no-op"); + let cleared = registry.remove_all().await.unwrap().unwrap(); + assert!(cleared.is_empty()); + assert!(registry.entries.load().is_none(), "empty state is stored as None"); + } + + #[tokio::test] + async fn retain_known_prunes_orphans_and_is_free_when_nothing_changes() { + let dir = TempDir::new().unwrap(); + let registry = registry_in(&dir); + registry.add_all(&["provider_a", "provider_b"], None, 100).await.unwrap(); + assert!(registry.retain_known(&["provider_a", "provider_b", "provider_c"]).await.unwrap().is_none()); + let pruned = registry.retain_known(&["provider_a"]).await.unwrap().unwrap(); + assert_eq!(pruned.keys().collect::>(), vec!["provider_a"]); + } + + #[tokio::test] + async fn failed_persist_leaves_runtime_unchanged() { + let dir = TempDir::new().unwrap(); + // A non-empty directory where the file should be makes the atomic rename fail. + let path = dir.path().join(MAINTENANCE_FILE); + std::fs::create_dir(&path).unwrap(); + std::fs::write(path.join("blocker"), b"x").unwrap(); + let registry = MaintenanceRegistry::new(path); + assert!(registry.add("provider_a", None, 100).await.is_err()); + assert!(!registry.is_in_maintenance("provider_a")); + } + + #[tokio::test] + async fn load_of_missing_file_is_empty() { + let dir = TempDir::new().unwrap(); + let registry = MaintenanceRegistry::load(dir.path().join(MAINTENANCE_FILE), &["provider_a"]).await; + assert!(registry.snapshot().is_empty()); + } + + #[tokio::test] + async fn load_moves_corrupt_file_aside() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join(MAINTENANCE_FILE); + std::fs::write(&path, b"{ not json").unwrap(); + let registry = MaintenanceRegistry::load(path.clone(), &["provider_a"]).await; + assert!(registry.snapshot().is_empty()); + assert!(!path.exists()); + assert!(dir.path().join("maintenance.json.corrupt").exists()); + } + + #[tokio::test] + async fn load_prunes_unknown_inputs_and_persists_the_result() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join(MAINTENANCE_FILE); + std::fs::write(&path, br#"{"gone":{"since":1},"provider_a":{"since":2}}"#).unwrap(); + let registry = MaintenanceRegistry::load(path.clone(), &["provider_a"]).await; + assert!(!registry.is_in_maintenance("gone")); + assert!(registry.is_in_maintenance("provider_a")); + let persisted: MaintenanceEntries = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(persisted.keys().collect::>(), vec!["provider_a"]); + } + + #[cfg(unix)] + #[tokio::test] + async fn unreadable_file_is_never_overwritten() { + use std::os::unix::fs::PermissionsExt; + let dir = TempDir::new().unwrap(); + let path = dir.path().join(MAINTENANCE_FILE); + std::fs::write(&path, br#"{"provider_a":{"since":1}}"#).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read(&path).is_ok() { + // Running as root: permissions do not stop reads, nothing to test. + return; + } + + let registry = MaintenanceRegistry::load(path.clone(), &["provider_a"]).await; + let write = registry.add("provider_b", None, 2).await; + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert!(write.is_err(), "state that could not be read must not be replaced"); + assert_eq!(std::fs::read(&path).unwrap(), br#"{"provider_a":{"since":1}}"#); + } + + #[tokio::test] + async fn read_only_load_never_touches_the_file() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join(MAINTENANCE_FILE); + std::fs::write(&path, br#"{"gone":{"since":1},"provider_a":{"since":2}}"#).unwrap(); + + let registry = MaintenanceRegistry::load_read_only(path.clone()).await; + + assert!(registry.is_in_maintenance("provider_a")); + assert!(registry.add("provider_b", None, 3).await.is_err(), "a read-only registry refuses writes"); + assert_eq!(std::fs::read(&path).unwrap(), br#"{"gone":{"since":1},"provider_a":{"since":2}}"#); + + let corrupt = dir.path().join("corrupt.json"); + std::fs::write(&corrupt, b"{ not json").unwrap(); + let registry = MaintenanceRegistry::load_read_only(corrupt.clone()).await; + assert!(registry.snapshot().is_empty()); + assert!(corrupt.exists(), "a read-only load never moves the file aside"); + } + + #[tokio::test] + async fn in_memory_registry_works_without_any_file() { + let registry = MaintenanceRegistry::in_memory(); + assert!(registry.add("provider_a", None, 1).await.unwrap().is_some()); + assert!(registry.is_in_maintenance("provider_a")); + assert!(registry.remove_all().await.unwrap().is_some()); + } + + #[tokio::test] + async fn gate_trait_delegates() { + let dir = TempDir::new().unwrap(); + let registry = Arc::new(registry_in(&dir)); + registry.add("provider_a", None, 1).await.unwrap(); + let gate: Arc = registry; + assert!(gate.is_in_maintenance("provider_a")); + } +} diff --git a/backend/core/src/model/mod.rs b/backend/core/src/model/mod.rs index 2f22c045b..aa0d5b422 100644 --- a/backend/core/src/model/mod.rs +++ b/backend/core/src/model/mod.rs @@ -18,6 +18,7 @@ pub mod auth_rejection; pub mod batch_result_collector; pub mod custom_stream_flags; pub mod fingerprint; +pub mod maintenance; pub mod playback; pub mod playlist_filter; pub mod playlist_key; @@ -34,8 +35,8 @@ pub mod xtream_response; pub use self::{ auth_rejection::*, batch_result_collector::*, config::*, custom_stream_flags::*, fingerprint::*, healthcheck::*, - http_clients::*, input_source::*, mapping::*, messaging::*, notification::*, playback::*, playlist_filter::*, - playlist_key::*, provider::*, proxy_redirect::*, stalker_record::*, stream_error::*, stream_history::*, - target_bouquet::*, update_guard::*, update_quality::*, update_task::*, xmltv::*, xtream::*, + http_clients::*, input_source::*, maintenance::*, mapping::*, messaging::*, notification::*, playback::*, + playlist_filter::*, playlist_key::*, provider::*, proxy_redirect::*, stalker_record::*, stream_error::*, + stream_history::*, target_bouquet::*, update_guard::*, update_quality::*, update_task::*, xmltv::*, xtream::*, }; pub use shared::model::xtream_const::*; diff --git a/backend/dvr/src/recording/mod.rs b/backend/dvr/src/recording/mod.rs index a8b7cf686..153f03d56 100644 --- a/backend/dvr/src/recording/mod.rs +++ b/backend/dvr/src/recording/mod.rs @@ -71,6 +71,7 @@ pub(crate) fn make_test_meta( measured_bytes: 0, completed_at: None, notification_markers: vec![], + provider_started: false, group: None, series_name: None, deleting_previous_state: None, diff --git a/backend/dvr/src/recording/recording_capacity.rs b/backend/dvr/src/recording/recording_capacity.rs index b562ccdfa..0303ef3e8 100644 --- a/backend/dvr/src/recording/recording_capacity.rs +++ b/backend/dvr/src/recording/recording_capacity.rs @@ -34,6 +34,9 @@ pub trait RecordingCapacityPort: Send + Sync { /// Fires when a connection is freed anywhere, so waiters can re-check. fn capacity_changed(&self) -> Arc; + + /// Whether the input is in maintenance; a recording on it must not start. + fn is_input_in_maintenance(&self, input_name: &str) -> bool; } #[cfg(test)] @@ -53,6 +56,7 @@ pub mod stub { /// Counts what the worker asked for, and hands out slots on request. pub struct StubCapacity { capacities: Mutex>, + maintenance: Mutex>, notify: Arc, pub acquires: AtomicUsize, pub releases: AtomicUsize, @@ -74,6 +78,7 @@ pub mod stub { fn with_capacity(in_use: usize, limit: usize) -> Arc { Arc::new(Self { capacities: Mutex::new(vec![(Arc::from("provider"), in_use, limit)]), + maintenance: Mutex::new(Vec::new()), notify: Arc::new(Notify::new()), acquires: AtomicUsize::new(0), releases: AtomicUsize::new(0), @@ -96,6 +101,15 @@ pub mod stub { pub fn acquire_count(&self) -> usize { self.acquires.load(Ordering::SeqCst) } pub fn release_count(&self) -> usize { self.releases.load(Ordering::SeqCst) } + + /// Marks an input as in maintenance (or not) for the next acquire decision. + pub fn set_maintenance(&self, input_name: &str, on: bool) { + let mut names = self.maintenance.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + names.retain(|name| name != input_name); + if on { + names.push(input_name.to_string()); + } + } } impl RecordingCapacityPort for StubCapacity { @@ -117,6 +131,14 @@ pub mod stub { } fn capacity_changed(&self) -> Arc { Arc::clone(&self.notify) } + + fn is_input_in_maintenance(&self, input_name: &str) -> bool { + self.maintenance + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .any(|name| name == input_name) + } } } diff --git a/backend/dvr/src/recording/recording_disk.rs b/backend/dvr/src/recording/recording_disk.rs index 54f691475..38fd6be19 100644 --- a/backend/dvr/src/recording/recording_disk.rs +++ b/backend/dvr/src/recording/recording_disk.rs @@ -247,6 +247,7 @@ mod tests { measured_bytes: measured, completed_at: None, notification_markers: Vec::new(), + provider_started: false, group: None, series_name: None, deleting_previous_state: None, diff --git a/backend/dvr/src/recording/recording_queue/control.rs b/backend/dvr/src/recording/recording_queue/control.rs index 5bf01a7b5..e2b85eb35 100644 --- a/backend/dvr/src/recording/recording_queue/control.rs +++ b/backend/dvr/src/recording/recording_queue/control.rs @@ -308,6 +308,7 @@ impl RecordingQueue { task.state = next; task.retry_attempts = 0; task.next_retry_at = None; + task.recording.provider_started = false; candidate.queue.push(task); Ok(Some(true)) } else { diff --git a/backend/dvr/src/recording/recording_quota.rs b/backend/dvr/src/recording/recording_quota.rs index 5173350c5..c755474fb 100644 --- a/backend/dvr/src/recording/recording_quota.rs +++ b/backend/dvr/src/recording/recording_quota.rs @@ -356,6 +356,7 @@ mod tests { measured_bytes: measured, completed_at: None, notification_markers: Vec::new(), + provider_started: false, group: None, series_name: None, deleting_previous_state: None, diff --git a/backend/dvr/src/recording/recording_retention.rs b/backend/dvr/src/recording/recording_retention.rs index 83aaf3a8b..1049833fd 100644 --- a/backend/dvr/src/recording/recording_retention.rs +++ b/backend/dvr/src/recording/recording_retention.rs @@ -284,6 +284,7 @@ mod tests { measured_bytes: 0, completed_at: Some(completed_at), notification_markers: Vec::new(), + provider_started: false, group: None, series_name: None, deleting_previous_state: None, diff --git a/backend/dvr/src/recording/recording_service/control.rs b/backend/dvr/src/recording/recording_service/control.rs index b7ddbcefd..3be2aa9c0 100644 --- a/backend/dvr/src/recording/recording_service/control.rs +++ b/backend/dvr/src/recording/recording_service/control.rs @@ -250,6 +250,7 @@ impl RecordingService { task.state = RecordingTaskState::Queued; task.retry_attempts = 0; task.next_retry_at = None; + task.recording.provider_started = false; candidate.queue.push(task); Ok(true) }, diff --git a/backend/dvr/src/recording/recording_transfer.rs b/backend/dvr/src/recording/recording_transfer.rs index f4f9fa340..92d445f51 100644 --- a/backend/dvr/src/recording/recording_transfer.rs +++ b/backend/dvr/src/recording/recording_transfer.rs @@ -28,6 +28,8 @@ const RECORDING_PROGRESS_UPDATE_INTERVAL: Duration = Duration::from_secs(5); const LIVE_CAPACITY_WINDOW_CLOSED: &str = "No provider capacity became available before the recording window closed"; +const MAINTENANCE_REASON: &str = "Input is in maintenance mode"; + const DOWNLOAD_PREEMPTED_REASON: &str = "Preempted by higher-priority foreground stream"; const RECORDING_PREEMPTED_REASON: &str = diff --git a/backend/dvr/src/recording/recording_transfer/provider.rs b/backend/dvr/src/recording/recording_transfer/provider.rs index 1783f551e..24824e722 100644 --- a/backend/dvr/src/recording/recording_transfer/provider.rs +++ b/backend/dvr/src/recording/recording_transfer/provider.rs @@ -22,6 +22,8 @@ pub(super) enum ProviderAcquireResult { Preempted, /// A live capture waited for capacity until its broadcast window closed. WindowClosed, + /// The input is in maintenance; the recording must not contact its provider. + Maintenance, } pub(super) fn background_download_should_wait( @@ -104,6 +106,7 @@ pub(super) async fn commit_acquired_download( active.error = None; active.paused = false; active.finished = false; + active.recording.provider_started = true; let notification = mark_recording_metadata_notification(&mut active.recording, LifecycleEvent::Started, None); Ok(Some(notification)) }) diff --git a/backend/dvr/src/recording/recording_transfer/tests.rs b/backend/dvr/src/recording/recording_transfer/tests.rs index 31a587d81..cc6a04122 100644 --- a/backend/dvr/src/recording/recording_transfer/tests.rs +++ b/backend/dvr/src/recording/recording_transfer/tests.rs @@ -1,10 +1,10 @@ use super::{ - acquire_result_after_wait, acquire_result_for_control, continue_after_pause, download_file, - ensure_recording_worker_running, finalize_http_transfer, finish_active_and_promote, http_transfer_path, - recording_deadline_instant, refresh_recording_progress, requeue_active_download_for_capacity_wait, - start_recording_scheduler, wait_for_provider_slot, DownloadExecutionResult, ProviderAcquireResult, QuotaGate, - RecordingNotificationPlan, DISK_GONE_BEFORE_START, DOWNLOAD_PREEMPTED_REASON, LIVE_CAPACITY_WINDOW_CLOSED, - QUOTA_EXCEEDED_DURING_TRANSFER, + acquire_result_after_wait, acquire_result_for_control, commit_acquired_download, continue_after_pause, + download_file, ensure_recording_worker_running, finalize_http_transfer, finish_active_and_promote, + http_transfer_path, recording_deadline_instant, refresh_recording_progress, + requeue_active_download_for_capacity_wait, start_recording_scheduler, wait_for_provider_slot, + DownloadExecutionResult, ProviderAcquireResult, QuotaGate, RecordingNotificationPlan, DISK_GONE_BEFORE_START, + DOWNLOAD_PREEMPTED_REASON, LIVE_CAPACITY_WINDOW_CLOSED, MAINTENANCE_REASON, QUOTA_EXCEEDED_DURING_TRANSFER, }; mod admission; diff --git a/backend/dvr/src/recording/recording_transfer/tests/admission.rs b/backend/dvr/src/recording/recording_transfer/tests/admission.rs index 5e095b9cb..735417791 100644 --- a/backend/dvr/src/recording/recording_transfer/tests/admission.rs +++ b/backend/dvr/src/recording/recording_transfer/tests/admission.rs @@ -4,7 +4,8 @@ use super::{ app_config_with_listener, bare_app_config, counting_ffmpeg, download_file, ensure_recording_worker_running, finish_active_and_promote, requeue_active_download_for_capacity_wait, scheduled_task, serve_range_fixture, slot_queue, spawn_count, vod_entry, wait_for_provider_slot, DownloadExecutionResult, QuotaGate, - RecordingNotificationPlan, DOWNLOAD_PREEMPTED_REASON, LIVE_CAPACITY_WINDOW_CLOSED, QUOTA_EXCEEDED_DURING_TRANSFER, + RecordingNotificationPlan, DOWNLOAD_PREEMPTED_REASON, LIVE_CAPACITY_WINDOW_CLOSED, MAINTENANCE_REASON, + QUOTA_EXCEEDED_DURING_TRANSFER, }; use crate::recording::{ recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, @@ -425,3 +426,113 @@ async fn a_live_capture_asks_for_capacity_once_runs_the_encoder_once_and_complet assert!(!crate::recording_partial_path(&capture.file_path).exists(), "and nothing was left staged"); assert_eq!(settled.recording.reserved_bytes, 0, "and it is not still holding disk"); } + +async fn start_worker_with(stub: &Arc, queue: &Arc) { + let capacity: Arc = Arc::clone(stub) as Arc; + ensure_recording_worker_running( + &bare_app_config(), + &RecordingConfig::from(&shared::model::RecordingConfigDto { + enabled: true, + ..shared::model::RecordingConfigDto::default() + }), + queue, + &NoopSink, + &capacity, + Path::new(crate::recording::recording_worker::FFMPEG_BINARY), + ) + .await + .expect("worker started"); +} + +async fn queued_live_capture(dir: &TempDir) -> Arc { queued_live_capture_with(dir, false).await } + +async fn queued_live_capture_with(dir: &TempDir, provider_started: bool) -> Arc { + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let now = chrono::Utc::now().timestamp(); + let mut capture = scheduled_task(RecordingKind::Live, now, 1_800); + capture.uuid = "live".to_string(); + capture.state = RecordingTaskState::Queued; + capture.input_name = Some(Arc::from("provider")); + capture.recording.provider_started = provider_started; + let capture = RecordingQueue::to_persisted(&capture); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.queue.push(capture.clone()); + Ok(()) + }) + .await + .expect("seed"); + queue +} + +#[tokio::test(start_paused = true)] +async fn a_capture_on_an_input_in_maintenance_fails_without_asking_for_a_slot() { + let dir = TempDir::new().expect("tempdir"); + let queue = queued_live_capture(&dir).await; + let stub = StubCapacity::with_room(); + stub.set_maintenance("provider", true); + + start_worker_with(&stub, &queue).await; + tokio::time::sleep(Duration::from_secs(5)).await; + + let finished = queue.finished.read().await; + let settled = finished.first().expect("the capture reached a terminal state"); + assert_eq!(settled.state, RecordingTaskState::Failed); + assert_eq!(settled.error.as_deref(), Some(MAINTENANCE_REASON), "and it says why"); + assert_eq!(stub.acquire_count(), 0, "no provider slot was requested"); + assert_eq!(stub.release_count(), 0); +} + +#[tokio::test(start_paused = true)] +async fn maintenance_on_another_input_does_not_block_the_capture() { + let dir = TempDir::new().expect("tempdir"); + let queue = queued_live_capture(&dir).await; + let stub = StubCapacity::with_room(); + stub.set_maintenance("some-other-input", true); + + start_worker_with(&stub, &queue).await; + tokio::time::sleep(Duration::from_secs(5)).await; + + assert_eq!(stub.acquire_count(), 1, "the capture asked for its slot as before"); +} + +#[tokio::test(start_paused = true)] +async fn a_capture_that_already_ran_keeps_retrying_during_maintenance() { + // A retry or a reload requeue of a recording that already ran is not a new start: + // running recordings, including their reconnects, are not affected by maintenance. + let dir = TempDir::new().expect("tempdir"); + let queue = queued_live_capture_with(&dir, true).await; + let stub = StubCapacity::with_room(); + stub.set_maintenance("provider", true); + + start_worker_with(&stub, &queue).await; + tokio::time::sleep(Duration::from_secs(5)).await; + + assert_eq!(stub.acquire_count(), 1, "the retry asked for its slot as before"); + let finished = queue.finished.read().await; + assert!( + finished.iter().all(|task| task.error.as_deref() != Some(MAINTENANCE_REASON)), + "a running recording is never failed for maintenance" + ); +} + +#[tokio::test(start_paused = true)] +async fn a_capture_waiting_for_capacity_fails_when_maintenance_starts() { + let dir = TempDir::new().expect("tempdir"); + let queue = queued_live_capture(&dir).await; + let stub = StubCapacity::full(); + + start_worker_with(&stub, &queue).await; + tokio::time::sleep(Duration::from_secs(5)).await; + assert_eq!(stub.acquire_count(), 1, "waiting for a slot"); + stub.set_maintenance("provider", true); + // What the capacity bridge does when a slot frees up somewhere. + for waiter in queue.slot_waiters.snapshots() { + let _ = queue.slot_waiters.signal_waiter(waiter.id); + } + tokio::time::sleep(Duration::from_secs(5)).await; + + let finished = queue.finished.read().await; + let settled = finished.first().expect("the capture reached a terminal state"); + assert_eq!(settled.error.as_deref(), Some(MAINTENANCE_REASON)); + assert_eq!(stub.acquire_count(), 1, "no further slot was requested"); +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/behavior.rs b/backend/dvr/src/recording/recording_transfer/tests/behavior.rs index a957942ab..84ae76155 100644 --- a/backend/dvr/src/recording/recording_transfer/tests/behavior.rs +++ b/backend/dvr/src/recording/recording_transfer/tests/behavior.rs @@ -1,9 +1,9 @@ use super::{ acquire_result_after_wait, acquire_result_for_control, app_config_with_listener, bare_app_config, - continue_after_pause, download_file, finalize_http_transfer, finish_active_and_promote, http_transfer_path, - read_request, recording_deadline_instant, requeue_active_download_for_capacity_wait, scheduled_task, - serve_range_fixture, slot_queue, start_recording_scheduler, wait_for_provider_slot, DownloadExecutionResult, - ProviderAcquireResult, RecordingNotificationPlan, + commit_acquired_download, continue_after_pause, download_file, finalize_http_transfer, finish_active_and_promote, + http_transfer_path, read_request, recording_deadline_instant, requeue_active_download_for_capacity_wait, + scheduled_task, serve_range_fixture, slot_queue, start_recording_scheduler, wait_for_provider_slot, + DownloadExecutionResult, ProviderAcquireResult, RecordingNotificationPlan, }; #[cfg(unix)] use super::{counting_ffmpeg, ensure_recording_worker_running}; @@ -99,6 +99,25 @@ async fn a_restart_requeue_consumes_the_restart_signal() { assert_eq!(*queue.worker("task").control_signal.read().await, RecordingControl::None); } +#[tokio::test] +async fn a_preempted_capture_stays_started_across_its_requeue() { + // Maintenance refuses only new starts; a capture that already held a provider slot + // must still count as started once a preemption put it back into the queue. + let queue = RecordingQueue::new(); + *queue.active.write().await = vec![scheduled_task(RecordingKind::Live, 0, 60)]; + assert!(!queue.active.read().await[0].recording.provider_started); + + commit_acquired_download(&queue, "task").await.expect("commit"); + assert!(queue.active.read().await[0].recording.provider_started); + + assert!(requeue_active_download_for_capacity_wait(&queue, "task", "preempted", false, None) + .await + .expect("requeue")); + let queued = queue.queue.lock().await; + let task = queued.iter().find(|task| task.uuid == "task").expect("requeued task"); + assert!(task.recording.provider_started); +} + #[test] fn a_slot_wait_maps_onto_the_acquisition_outcome() { // A wait that ends without an outcome ran into the live window's end; diff --git a/backend/dvr/src/recording/recording_transfer/tests/support.rs b/backend/dvr/src/recording/recording_transfer/tests/support.rs index 407315d9a..b29fb821a 100644 --- a/backend/dvr/src/recording/recording_transfer/tests/support.rs +++ b/backend/dvr/src/recording/recording_transfer/tests/support.rs @@ -238,6 +238,8 @@ impl RecordingCapacityPort for LimitedCapacity { } fn capacity_changed(&self) -> Arc { self.notify.clone() } + + fn is_input_in_maintenance(&self, _input_name: &str) -> bool { false } } #[cfg(unix)] diff --git a/backend/dvr/src/recording/recording_transfer/worker.rs b/backend/dvr/src/recording/recording_transfer/worker.rs index 212dc1477..bf2a89422 100644 --- a/backend/dvr/src/recording/recording_transfer/worker.rs +++ b/backend/dvr/src/recording/recording_transfer/worker.rs @@ -8,7 +8,7 @@ use super::{ requeue_active_download_for_retry, set_active_download_state, should_exit_worker_after_preempt, spawn_recording_notification_after_persist, take_active, wait_for_provider_slot, write_completion_sidecar, ProviderAcquireResult, ProviderCapacities, QuotaGate, RetryCommit, DOWNLOAD_PREEMPTED_REASON, - LIVE_CAPACITY_WINDOW_CLOSED, RECORDING_PROGRESS_UPDATE_INTERVAL, + LIVE_CAPACITY_WINDOW_CLOSED, MAINTENANCE_REASON, RECORDING_PROGRESS_UPDATE_INTERVAL, }; use crate::recording::{ recording_capacity::RecordingCapacityPort, @@ -147,14 +147,14 @@ async fn start_recording_worker( 'worker: loop { // One read: the uuid, pause flag and live window must describe // the same task. - let active_head = dq - .active - .read() - .await - .iter() - .find(|task| task.uuid == worker_uuid) - .map(|download| (download.paused, recording_deadline_instant(download))); - if let Some((paused, window_deadline)) = active_head { + let active_head = + dq.active.read().await.iter().find(|task| task.uuid == worker_uuid).map(|download| { + // A retry or a requeue of a recording that already ran is not a new + // start; maintenance only refuses new ones. + let first_start = !download.recording.provider_started; + (download.paused, recording_deadline_instant(download), first_start) + }); + if let Some((paused, window_deadline, first_start)) = active_head { if paused { break; } @@ -176,6 +176,11 @@ async fn start_recording_worker( // concurrent recording starts respect background limits. let _capacity = dq.capacity_guard.lock().await; let capacities = capacity.capacities_for_input(&input_name).await; + // Checked right before every slot request, after every await + // of this attempt, so nothing has to be released. + if first_start && capacity.is_input_in_maintenance(&input_name) { + break ProviderAcquireResult::Maintenance; + } let live_config = app_config.config.load(); let capacity_cfg = live_config.recording().unwrap_or(&download_cfg); if background_download_should_wait(priority, &capacities, capacity_cfg) { @@ -259,6 +264,17 @@ async fn start_recording_worker( } continue; } + ProviderAcquireResult::Maintenance => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + fail_active_download(&dq, &worker_uuid, MAINTENANCE_REASON).await, + "input maintenance", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + continue; + } ProviderAcquireResult::Paused => { break; } diff --git a/backend/dvr/src/recording/recording_worker_runner.rs b/backend/dvr/src/recording/recording_worker_runner.rs index 05ef3916e..51d39a1a5 100644 --- a/backend/dvr/src/recording/recording_worker_runner.rs +++ b/backend/dvr/src/recording/recording_worker_runner.rs @@ -342,6 +342,7 @@ mod tests { measured_bytes: measured, completed_at: Some(completed_at), notification_markers: Vec::new(), + provider_started: false, group: None, series_name: None, deleting_previous_state: None, diff --git a/backend/hls/src/availability/tests.rs b/backend/hls/src/availability/tests.rs index 6bd1385ee..bf7f3bd80 100644 --- a/backend/hls/src/availability/tests.rs +++ b/backend/hls/src/availability/tests.rs @@ -118,6 +118,7 @@ fn runtime_custom_responses_with_low_priority(low_priority_bytes: &[u8]) -> Arc< user_account_expired: Some(TransportStreamBuffer::new(ACCOUNT_EXPIRED_ASSET_BYTES.to_vec())), panel_api_provisioning: None, hls_session_or_lease_expired: Some(TransportStreamBuffer::new(SESSION_EXPIRED_ASSET_BYTES.to_vec())), + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), }) } diff --git a/backend/hls/src/origin.rs b/backend/hls/src/origin.rs index 1c533730f..e18872fc3 100644 --- a/backend/hls/src/origin.rs +++ b/backend/hls/src/origin.rs @@ -1040,64 +1040,57 @@ pub async fn finish_hls_origin_account_io( refresh_reservation: bool, ) { let binding = guard.binding().clone(); - let mut provider_resource_to_release = None; + let mut session = session.write().await; + + let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); + let confirmed_ttl_secs = session.activity.last_delivered_media_at_ms.map_or(0, |last| { + last.saturating_add(origin_io.reservation_ttl_secs.saturating_mul(1000)).saturating_sub(now_ms).div_ceil(1000) + }); + + // Decrement directly on guard.lease, not session.origin_account_io_lease. + // The session may have been rebound or replaced between I/O start and finish. + // Only clear the session reference if it still points to this exact lease instance. + guard.disarm(); let mut should_refresh_reservation = false; - let confirmed_ttl_secs; - { - let mut session = session.write().await; - - let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); - confirmed_ttl_secs = session.activity.last_delivered_media_at_ms.map_or(0, |last| { - last.saturating_add(origin_io.reservation_ttl_secs.saturating_mul(1000)) - .saturating_sub(now_ms) - .div_ceil(1000) - }); - - // Decrement directly on guard.lease, not session.origin_account_io_lease. - // The session may have been rebound or replaced between I/O start and finish. - // Only clear the session reference if it still points to this exact lease instance. - guard.disarm(); - if guard.lease.active_io_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel) == 1 { - provider_resource_to_release = guard.lease.take_resource(); - if session - .origin_account_io_lease - .as_ref() - .is_some_and(|session_lease| std::sync::Arc::ptr_eq(session_lease, &guard.lease)) - { - session.origin_account_io_lease = None; - } - if refresh_reservation && confirmed_ttl_secs > 0 { - should_refresh_reservation = true; - } + if guard.lease.active_io_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel) == 1 { + let provider_resource_to_release = guard.lease.take_resource(); + if session + .origin_account_io_lease + .as_ref() + .is_some_and(|session_lease| std::sync::Arc::ptr_eq(session_lease, &guard.lease)) + { + session.origin_account_io_lease = None; } - - if let Some(current) = session.origin_account_binding.as_mut().filter(|current| { - current.is_active() - && current.account_name == binding.account_name - && current.session_owner == binding.session_owner - && current.generation == binding.generation - }) { - let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); - current.last_origin_io_at_ms = Some(now_ms); + should_refresh_reservation = refresh_reservation && confirmed_ttl_secs > 0; + if let Some((provider_handle, cleanup_permit)) = provider_resource_to_release { if should_refresh_reservation { - current.last_reservation_refresh_at_ms = Some(now_ms); + if let Some(request_id) = binding.playback_request_id { + origin_io.ctx.active_provider.refresh_playback_lease( + &binding.account_name, + &PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }, + confirmed_ttl_secs, + ); + } } + // Released synchronously and still under the session lock: the next I/O of + // this session reserves its slot under the same lock, so it can never see the + // lease gone while the provider still counts the old connection. A deferred + // release lets it race for a new slot, which an account at its connection + // limit answers with `Exhausted`. + origin_io.ctx.connection_manager.release_provider_handle(Some(provider_handle)); + drop(cleanup_permit); } } - if let Some((provider_handle, cleanup_permit)) = provider_resource_to_release { - let mut handle_guard = ManagedHandleGuard::new(provider_handle, cleanup_permit); + if let Some(current) = session.origin_account_binding.as_mut().filter(|current| { + current.is_active() + && current.account_name == binding.account_name + && current.session_owner == binding.session_owner + && current.generation == binding.generation + }) { + current.last_origin_io_at_ms = Some(now_ms); if should_refresh_reservation { - if let Some(request_id) = binding.playback_request_id { - origin_io.ctx.active_provider.refresh_playback_lease( - &binding.account_name, - &PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }, - confirmed_ttl_secs, - ); - } - } - if let Some((handle, permit)) = handle_guard.take_resource() { - permit.send(CleanupEvent::ReleaseProviderHandle { handle: Some(handle) }); + current.last_reservation_refresh_at_ms = Some(now_ms); } } } diff --git a/backend/hls/src/refresh/tests/support.rs b/backend/hls/src/refresh/tests/support.rs index 07ce920f9..07323d3cc 100644 --- a/backend/hls/src/refresh/tests/support.rs +++ b/backend/hls/src/refresh/tests/support.rs @@ -945,6 +945,7 @@ pub(in crate::refresh::tests) fn critical_handoff_app_config() -> Arc user_account_expired: None, panel_api_provisioning: None, hls_session_or_lease_expired: None, + maintenance: None, panel_api_provisioning_hls_segments: Vec::new(), }))); app_config diff --git a/backend/hls/src/runtime_custom_tail.rs b/backend/hls/src/runtime_custom_tail.rs index 9bcd22e5d..77e9120e6 100644 --- a/backend/hls/src/runtime_custom_tail.rs +++ b/backend/hls/src/runtime_custom_tail.rs @@ -101,7 +101,8 @@ impl HlsRuntimeCustomTailReason { CustomVideoStreamType::ProviderConnectionsExhausted => Some(Self::ProviderConnectionsExhausted), CustomVideoStreamType::UserAccountExpired => Some(Self::UserAccountExpired), CustomVideoStreamType::HlsSessionOrLeaseExpired => Some(Self::SessionOrLeaseExpired), - CustomVideoStreamType::Provisioning => None, + // Maintenance only answers new entries; it never ends a running session as a tail. + CustomVideoStreamType::Provisioning | CustomVideoStreamType::Maintenance => None, } } } diff --git a/backend/metadata/src/ctx.rs b/backend/metadata/src/ctx.rs index a01f6d3ae..c448f1382 100644 --- a/backend/metadata/src/ctx.rs +++ b/backend/metadata/src/ctx.rs @@ -8,7 +8,7 @@ use shared::model::EventSink; use std::sync::Arc; -use tuliprox_core::model::{AppConfig, HttpClients, UpdateGuard}; +use tuliprox_core::model::{AppConfig, HttpClients, MaintenanceGate, UpdateGuard}; use tuliprox_repository::PlaylistStorageState; use tuliprox_session::{ActiveProviderManager, ConnectionManager, EventManager}; @@ -29,6 +29,8 @@ pub struct MetadataUpdateCtx { pub update_guard: UpdateGuard, /// Outbound clients, swapped when the proxy configuration changes. pub http_clients: Arc, + /// Inputs in maintenance; their tasks wait instead of contacting the provider. + pub maintenance: Arc, } /// The instantiation the long-lived worker stores. diff --git a/backend/metadata/src/manager.rs b/backend/metadata/src/manager.rs index 3f4d7a733..b81ae5975 100644 --- a/backend/metadata/src/manager.rs +++ b/backend/metadata/src/manager.rs @@ -17,6 +17,8 @@ const TASK_ERR_PREEMPTED: &str = "Task preempted"; const TASK_ERR_UPDATE_IN_PROGRESS: &str = "Playlist update in progress"; +const TASK_ERR_MAINTENANCE: &str = "Input in maintenance"; + // Per-task execution timeout. ffprobe defaults to 60 s; allow extra headroom for network // fetches, TMDB lookups, and B+Tree writes before declaring the task stuck. const PROBE_TASK_TIMEOUT_SECS: u64 = 30; diff --git a/backend/metadata/src/manager/retry.rs b/backend/metadata/src/manager/retry.rs index 12322eec1..5f69b11a8 100644 --- a/backend/metadata/src/manager/retry.rs +++ b/backend/metadata/src/manager/retry.rs @@ -1,8 +1,8 @@ use super::{ load_metadata_retry_states_from_disk, persist_metadata_retry_state_to_disk, spawn_blocking_limited, InputWorker, MetadataUpdateManager, MetadataUpdateRuntimeSettings, ScopedTaskKey, TaskKey, METADATA_RETRY_STATE_FILE, - RETRY_STATE_MIN_TTL_SECS, RETRY_STATE_PRUNE_INTERVAL_SECS, TASK_ERR_NO_CONNECTION, TASK_ERR_PREEMPTED, - TASK_ERR_UPDATE_IN_PROGRESS, + RETRY_STATE_MIN_TTL_SECS, RETRY_STATE_PRUNE_INTERVAL_SECS, TASK_ERR_MAINTENANCE, TASK_ERR_NO_CONNECTION, + TASK_ERR_PREEMPTED, TASK_ERR_UPDATE_IN_PROGRESS, }; use crate::ctx::MetadataUpdateCtx; use log::warn; @@ -451,7 +451,10 @@ impl InputWorker { } pub(super) fn is_transient_worker_error(message: &str) -> bool { - message == TASK_ERR_UPDATE_IN_PROGRESS || message == TASK_ERR_PREEMPTED || message == TASK_ERR_NO_CONNECTION + message == TASK_ERR_UPDATE_IN_PROGRESS + || message == TASK_ERR_PREEMPTED + || message == TASK_ERR_NO_CONNECTION + || message == TASK_ERR_MAINTENANCE } #[inline] diff --git a/backend/metadata/src/manager/task_processing.rs b/backend/metadata/src/manager/task_processing.rs index 4f3b1a701..e6673d85f 100644 --- a/backend/metadata/src/manager/task_processing.rs +++ b/backend/metadata/src/manager/task_processing.rs @@ -1,5 +1,5 @@ use super::{ - spawn_blocking_limited, DbHandle, InputWorker, RetryDomain, TaskKey, PROBE_TASK_TIMEOUT_SECS, + spawn_blocking_limited, DbHandle, InputWorker, RetryDomain, TaskKey, PROBE_TASK_TIMEOUT_SECS, TASK_ERR_MAINTENANCE, TASK_ERR_NO_CONNECTION, TASK_ERR_PREEMPTED, TASK_ERR_UPDATE_IN_PROGRESS, }; use crate::ctx::MetadataUpdateCtx; @@ -117,6 +117,11 @@ impl InputWorker { return Err(shared::error::TuliproxError::Config(format!("Input {input_name} is disabled"))); } + // Waits instead of failing: the task is requeued and resolved once maintenance ends. + if ctx.maintenance.is_in_maintenance(&input_base.name) { + return Err(shared::error::TuliproxError::Config(TASK_ERR_MAINTENANCE.to_string())); + } + // Background metadata/probe tasks are low-priority. // Never run them while a foreground playlist update is active. if let Some(guard) = ctx.update_guard.try_playlist() { diff --git a/backend/metadata/src/manager/tests/behavior.rs b/backend/metadata/src/manager/tests/behavior.rs index c8daed1de..2e91c80e5 100644 --- a/backend/metadata/src/manager/tests/behavior.rs +++ b/backend/metadata/src/manager/tests/behavior.rs @@ -1,6 +1,7 @@ use super::{ create_test_worker, InputWorker, MetadataUpdateManager, MetadataUpdateRuntimeSettings, PendingTask, RetryDomain, - ScopedTaskKey, SubmitTaskResult, TaskKey, TASK_ERR_NO_CONNECTION, TASK_ERR_PREEMPTED, TASK_ERR_UPDATE_IN_PROGRESS, + ScopedTaskKey, SubmitTaskResult, TaskKey, TASK_ERR_MAINTENANCE, TASK_ERR_NO_CONNECTION, TASK_ERR_PREEMPTED, + TASK_ERR_UPDATE_IN_PROGRESS, }; use dashmap::DashMap; use shared::{ @@ -824,3 +825,22 @@ fn playlist_trigger_keeps_info_changes() { assert!(InputWorker::should_trigger_playlist_update_for_task(&task, true)); assert!(!InputWorker::should_trigger_playlist_update_for_task(&task, false)); } + +#[test] +fn maintenance_is_a_transient_worker_error() { + // Transient errors are requeued without counting an attempt, so the item is + // resolved after maintenance ends instead of being marked as failed. + assert!(InputWorker::is_transient_worker_error(TASK_ERR_MAINTENANCE)); +} + +#[test] +fn maintenance_check_precedes_probe_connection() { + let source = include_str!("../task_processing.rs"); + let start = source.find("pub(super) async fn process_task_static").expect("process_task_static"); + let body = &source[start..]; + // Methods are indented once, so their closing brace is `\n }\n`. + let body = &body[..body.find("\n }\n").expect("function end")]; + let gate = body.find("if ctx.maintenance.is_in_maintenance(").expect("maintenance gate"); + let probe = body.find("acquire_connection_for_probe(").expect("probe acquisition"); + assert!(gate < probe, "no provider connection may be taken for an input in maintenance"); +} diff --git a/backend/processing/src/epg.rs b/backend/processing/src/epg.rs index f2493082b..57e9ec723 100644 --- a/backend/processing/src/epg.rs +++ b/backend/processing/src/epg.rs @@ -11,7 +11,7 @@ use std::{ path::{Path, PathBuf}, }; use tuliprox_core::{ - model::{ConfigInput, EpgSource, EpgSourceType, PersistedEpgSource, PersistedEpgSourceKind}, + model::{ConfigInput, EpgConfig, EpgSource, EpgSourceType, PersistedEpgSource, PersistedEpgSourceKind}, utils::{add_prefix_to_filename, prepare_file_path, request}, }; use tuliprox_repository::get_input_storage_path; @@ -191,6 +191,9 @@ pub async fn get_xmltv( ) -> (Option, Vec) { match &input.epg { None => (None, vec![]), + Some(epg_config) if ctx.is_input_in_maintenance(&input.name) => { + persisted_epg_for_maintenance(ctx, input, epg_config, storage_dir).await + } Some(epg_config) => { let mut errors = vec![]; let mut file_paths = vec![]; @@ -226,6 +229,35 @@ pub async fn get_xmltv( } } +/// The EPG files fetched last, without contacting any source and without cleanup: +/// deleting "unlisted" files here would throw away exactly what maintenance keeps. +async fn persisted_epg_for_maintenance( + ctx: &PlaylistProcessingContext, + input: &ConfigInput, + epg_config: &EpgConfig, + storage_dir: &str, +) -> (Option, Vec) { + debug!("Input '{}' is in maintenance; using its persisted epg", input.name); + let mut errors = vec![]; + let mut file_paths = vec![]; + for epg_source in &epg_config.sources { + let Ok(path) = get_input_raw_epg_file_path(epg_source, input, storage_dir).await else { + continue; + }; + if tokio::fs::metadata(&path).await.is_ok_and(|metadata| metadata.is_file()) { + match persisted_source_from_config(epg_source, path) { + Ok(persisted) => file_paths.push(persisted), + Err(err) => errors.push(err), + } + } + } + if file_paths.is_empty() { + (None, errors) + } else { + (Some(TVGuide::new(file_paths).with_file_locks(std::sync::Arc::clone(&ctx.config.file_locks))), errors) + } +} + fn persisted_source_from_config( epg_source: &EpgSource, file_path: PathBuf, diff --git a/backend/processing/src/processor/playlist/ingest.rs b/backend/processing/src/processor/playlist/ingest.rs index 10afe8381..55081cdd1 100644 --- a/backend/processing/src/processor/playlist/ingest.rs +++ b/backend/processing/src/processor/playlist/ingest.rs @@ -114,6 +114,8 @@ pub struct PlaylistProcessingContext, pub(crate) library_update_mode: LibraryUpdateMode, + /// Inputs in maintenance: their persisted state is reused, their provider is not contacted. + pub maintenance: Option>, } #[cfg(test)] diff --git a/backend/processing/src/processor/playlist/ingest/acquisition.rs b/backend/processing/src/processor/playlist/ingest/acquisition.rs index eead204c1..f1c7d8fda 100644 --- a/backend/processing/src/processor/playlist/ingest/acquisition.rs +++ b/backend/processing/src/processor/playlist/ingest/acquisition.rs @@ -306,6 +306,12 @@ pub(crate) async fn download_input Vec { let Some(aliases) = input.get_enabled_aliases() else { return vec![] }; + // Maintenance is keyed by the parent input; an alias carries its own name and would + // otherwise contact the provider. Its persisted playlist stays in place. + if ctx.is_input_in_maintenance(&input.name) { + debug!("Input '{}' is in maintenance; skipping its M3U alias downloads", input.name); + return vec![]; + } let mut errors = Vec::new(); for alias in aliases { diff --git a/backend/processing/src/processor/playlist/ingest/jobs.rs b/backend/processing/src/processor/playlist/ingest/jobs.rs index a5a375ccf..9bb84e185 100644 --- a/backend/processing/src/processor/playlist/ingest/jobs.rs +++ b/backend/processing/src/processor/playlist/ingest/jobs.rs @@ -466,11 +466,16 @@ impl Clone for PlaylistProcessingCo had_quality_rejections: Arc::clone(&self.had_quality_rejections), input_refresh: self.input_refresh, library_update_mode: self.library_update_mode, + maintenance: self.maintenance.clone(), } } } impl PlaylistProcessingContext { + pub fn is_input_in_maintenance(&self, input_name: &str) -> bool { + self.maintenance.as_ref().is_some_and(|gate| gate.is_in_maintenance(input_name)) + } + #[must_use] pub fn refresh_policy(&self, input_id: u16) -> InputRefreshPolicy { self.input_refresh diff --git a/backend/processing/src/processor/playlist/mod.rs b/backend/processing/src/processor/playlist/mod.rs index 914517285..7f0891f5d 100644 --- a/backend/processing/src/processor/playlist/mod.rs +++ b/backend/processing/src/processor/playlist/mod.rs @@ -146,6 +146,7 @@ pub struct ProcessingRun< acquired_permit: Option, input_refresh: Option, library_update_mode: LibraryUpdateMode, + maintenance: Option>, } impl ProcessingRun { @@ -178,6 +179,7 @@ impl ProcessingRun acquired_permit: self.acquired_permit, input_refresh: self.input_refresh, library_update_mode: self.library_update_mode, + maintenance: self.maintenance, } } @@ -255,6 +258,7 @@ impl acquired_permit: self.acquired_permit, input_refresh: self.input_refresh, library_update_mode: self.library_update_mode, + maintenance: self.maintenance, } } @@ -291,6 +295,13 @@ impl self } + /// Inputs taken out of service; their provider is not contacted during this run. + #[must_use] + pub fn with_maintenance(mut self, gate: Arc) -> Self { + self.maintenance = Some(gate); + self + } + /// Applies one manual policy override without changing the input configuration. #[must_use] pub fn with_input_refresh(mut self, input_refresh: impl Into>) -> Self { @@ -319,6 +330,7 @@ pub async fn exec_processing, processed_fpl: &mut FetchedPlaylist<'_>, ) { + // Resolve and probe talk to the provider; an input in maintenance keeps what it has. + if ctx.is_input_in_maintenance(&processed_fpl.input.name) { + return; + } playlist_resolve_series(ctx, target, errors, pipe, provider_fpl, processed_fpl).await; playlist_resolve_vod(ctx, target, errors, provider_fpl, processed_fpl).await; playlist_probe(ctx, target, processed_fpl).await; diff --git a/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs b/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs index edf1f8b3e..8668e1a77 100644 --- a/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs +++ b/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs @@ -73,6 +73,7 @@ pub(super) fn processing_context( stalker_refresh_mode: StalkerRefreshMode::Complete, partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)), had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)), + maintenance: None, input_refresh: None, library_update_mode: LibraryUpdateMode::ExistingCatalog, } diff --git a/backend/processing/src/processor/playlist/tests/mapping_stage.rs b/backend/processing/src/processor/playlist/tests/mapping_stage.rs index 06402ee67..395391c8b 100644 --- a/backend/processing/src/processor/playlist/tests/mapping_stage.rs +++ b/backend/processing/src/processor/playlist/tests/mapping_stage.rs @@ -97,6 +97,7 @@ fn processing_context() -> PlaylistProcessingContext { stalker_refresh_mode: StalkerRefreshMode::Complete, partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)), had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)), + maintenance: None, } } diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs index 2d8c4c923..47c302c4b 100644 --- a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs @@ -240,6 +240,7 @@ fn processing_context_with_events(storage_dir: &Path, events: E) - stalker_refresh_mode: StalkerRefreshMode::Complete, partial_refresh: Arc::new(AtomicBool::new(false)), had_quality_rejections: Arc::new(AtomicBool::new(false)), + maintenance: None, } } @@ -1617,3 +1618,104 @@ mod m3u_update_quality; mod manual_update_integration; mod persisted_input_status; mod staged_completion; + +struct FixedGate(&'static str); + +impl tuliprox_core::model::MaintenanceGate for FixedGate { + fn is_in_maintenance(&self, input_name: &str) -> bool { input_name == self.0 } +} + +fn quality_95() -> ConfigInputUpdateQualityDto { ConfigInputUpdateQualityDto { live: 95, vod: 95, series: 95 } } + +#[tokio::test] +async fn maintenance_input_uses_persisted_playlist_without_download() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 2, 2]); + let mut ctx = processing_context(temp.path()); + // cache_duration_seconds = 0: without maintenance this input is downloaded. + let input = test_input(&server.base_url, quality_95(), 0, &[]); + seed_baseline(&ctx, &input).await; + ctx.maintenance = Some(Arc::new(FixedGate("quality-provider"))); + + let mut result = download_input(&ctx, &input, false).await; + + assert!(result.errors.is_empty(), "maintenance must not fail the input: {:?}", result.errors); + let baseline_channels: usize = baseline_groups().iter().map(|group| group.channels.len()).sum(); + assert_eq!(result.source.get_channel_count(), baseline_channels, "the persisted playlist is served"); + assert_requested_actions(server.finish(), &[]); +} + +#[tokio::test] +async fn maintenance_input_without_cache_contributes_nothing() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 2, 2]); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, quality_95(), 0, &[]); + ctx.maintenance = Some(Arc::new(FixedGate("quality-provider"))); + + let mut result = download_input(&ctx, &input, false).await; + + assert!(result.errors.is_empty(), "a missing cache is not an error in maintenance: {:?}", result.errors); + assert_eq!(result.source.get_channel_count(), 0); + assert_requested_actions(server.finish(), &[]); +} + +#[tokio::test] +async fn input_not_in_maintenance_still_downloads() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 2, 2]); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, quality_95(), 0, &[]); + seed_baseline(&ctx, &input).await; + ctx.maintenance = Some(Arc::new(FixedGate("some-other-input"))); + + let mut result = download_input(&ctx, &input, false).await; + + assert!(result.errors.is_empty(), "{:?}", result.errors); + assert_eq!(result.source.get_channel_count(), 6); + assert!(!server.finish().is_empty(), "the provider was contacted"); +} + +#[tokio::test] +async fn maintenance_input_skips_epg_download_and_keeps_existing_file() { + use tuliprox_core::model::{EpgConfig, EpgSource, EpgSourceType}; + + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 2, 2]); + let mut ctx = processing_context(temp.path()); + let mut input = (*test_input(&server.base_url, quality_95(), 0, &[])).clone(); + let epg_source = EpgSource { + source_type: EpgSourceType::Xmltv, + url: format!("{}/xmltv.php", server.base_url), + priority: 0, + logo_override: false, + channel_id: None, + channel_title: None, + match_names: Vec::new(), + ics: None, + }; + input.epg = Some(EpgConfig { sources: vec![epg_source.clone()], smart_match: None }); + let storage_dir = temp.path().to_string_lossy().into_owned(); + let epg_path = + crate::epg::get_input_raw_epg_file_path(&epg_source, &input, &storage_dir).await.expect("epg file path"); + tokio::fs::write(&epg_path, "").await.expect("seed persisted epg"); + ctx.maintenance = Some(Arc::new(FixedGate("quality-provider"))); + + let (guide, errors) = crate::epg::get_xmltv(&ctx, &input, None, &storage_dir).await; + + assert!(errors.is_empty(), "{errors:?}"); + assert!(guide.is_some(), "the persisted epg is used"); + assert!(epg_path.exists(), "maintenance must not clean up the persisted epg"); + assert_requested_actions(server.finish(), &[]); +} + +#[test] +fn resolve_stage_checks_maintenance_before_any_provider_work() { + let source = include_str!("../target.rs"); + let start = source.find("pub(crate) async fn playlist_resolve<").expect("playlist_resolve"); + let body = &source[start..]; + let body = &body[..body.find("\n}\n").expect("function end")]; + let gate = body.find("if ctx.is_input_in_maintenance(").expect("maintenance gate in playlist_resolve"); + let series = body.find("playlist_resolve_series(").expect("series resolve call"); + assert!(gate < series, "the gate must run before resolve and probe"); +} diff --git a/docs/src/configuration/config.md b/docs/src/configuration/config.md index a59dd6821..0a040a683 100644 --- a/docs/src/configuration/config.md +++ b/docs/src/configuration/config.md @@ -269,7 +269,8 @@ playlist_manager:playlist.read,playlist.write,source.read Available Permissions: `config.read/write`, `source.read/write`, `user.read/write`, `playlist.read/write`, `library.read/write`, -`system.read/write`, `epg.read/write`, `recording.read/create/manage/delete`. Note: Write does not imply Read. A group must explicitly +`system.read/write`, `epg.read/write`, `recording.read/create/manage/delete`, `maintenance.read/write`. +Note: Write does not imply Read. A group must explicitly grant both if users need to view and edit content. @@ -481,6 +482,7 @@ negative pattern does, so `["*", "!system.info"]` reads the way it looks. | `user.connection.denied` | warn | A user was refused a connection because their limits were full. | | `recording.queue.changed` | info | The recording queue changed. | | `recording.rules.changed` | info | The recording rule set changed. | +| `maintenance.changed` | warn | An input was put into or taken out of maintenance mode. | | `scheduled_task.failed` | error | A scheduled task could not complete. | | `notification.dead_lettered` | error | A notification was permanently undeliverable and has been dropped. | | `user.created` | info | An API-proxy user account was created. | @@ -1197,6 +1199,7 @@ custom_stream_response_timeout_secs: 20 * `user_account_expired.ts` (User's `exp_date` reached) * `panel_api_provisioning.ts` (Loops while a new Provider Account is generated via Panel API) * `hls_session_or_lease_expired.ts` (Shared-HLS session or access lease expired; restart stream) +* `maintenance.ts` (The input is in maintenance mode; see [Maintenance mode](../rest-api-cookbook.md#maintenance-mode)) > **Note**: These Video files are all available in the docker image. diff --git a/docs/src/features.md b/docs/src/features.md index c6125fce9..b58bc8722 100644 --- a/docs/src/features.md +++ b/docs/src/features.md @@ -37,6 +37,7 @@ That makes it usable both for IPTV players and for media-server-oriented workflo - enforce user connection limits - prioritize higher-value sessions over lower-priority traffic - serve custom fallback videos for failure cases +- put inputs into maintenance mode: new streams get a maintenance video, the provider is not contacted - persist optional stream history for per-variant connect/disconnect, startup-failure, and reconnect telemetry - aggregate optional QoS snapshots from stream history to rank equivalent channel variants by reliability @@ -63,7 +64,8 @@ Tuliprox also includes: Tuliprox supports role-based access control (RBAC) for the Web UI: -- fine-grained permissions across 7 domains (config, source, user, playlist, library, system, epg) +- fine-grained permissions across 9 domains (config, source, user, playlist, library, system, epg, recording, + maintenance) - custom groups with configurable permission sets - user-to-group assignments with union-based permission resolution - compact bitmask encoding in JWT claims for low-overhead permission checks diff --git a/docs/src/operator/dvr.md b/docs/src/operator/dvr.md index 570c02871..6b819944a 100644 --- a/docs/src/operator/dvr.md +++ b/docs/src/operator/dvr.md @@ -428,6 +428,12 @@ constant; pre-bump tokens become stale: Operators do **not** need to manually invalidate tokens on a schema bump; the next sign-in issues a token with the current schema and the user's derived `subject_id`. +## 10.1 Maintenance mode + +A recording whose input is in maintenance when it would start is not started: it fails with +`Input is in maintenance mode` without requesting a provider slot. Running recordings, including their internal +reconnects, continue. Nothing is retried automatically when maintenance ends; use **Retry** on the failed task. + ## 11. The removed `/file/record` route `POST /file/record` and the whole `/api/v1/file/download/*` family are diff --git a/docs/src/rest-api-cookbook.md b/docs/src/rest-api-cookbook.md index c8b8f7226..9163f10aa 100644 --- a/docs/src/rest-api-cookbook.md +++ b/docs/src/rest-api-cookbook.md @@ -349,6 +349,58 @@ Notes: are `null` for semantic errors without a source position (e.g. an invalid regex value). - The preview reads the target's already-processed playlist; it never contacts providers or triggers an update. +## Maintenance mode + +An input can be taken out of service while its provider is down. While an input is in maintenance: + +- every **new** stream request for it (live, VOD, series, catch-up, HLS entry, redirect mode) gets the + `maintenance.ts` custom video instead of a provider connection, +- playlist and EPG updates keep serving the last persisted playlist and EPG without contacting the provider, +- probes, metadata resolution and Stalker refreshes for the input are paused, +- DVR recordings that would start fail with `Input is in maintenance mode`; use **Retry** afterwards. + +Running streams, seeks within them, and running recordings continue. The state is stored in +`/maintenance.json` and survives restarts. A corrupt file is moved aside as +`maintenance.json.corrupt` and the server starts with no input in maintenance. A file that cannot be read at all +(for example because of its permissions) stays in place; changes are then refused with `500` instead of overwriting +it. A one-shot CLI update reads the file but never changes it. Entries for inputs that are removed +from `source.yml` are dropped on the next source reload. + +The dashboard shows a **Maintenance** card that turns red while any input is in maintenance. + +```bash +#!/bin/bash + +BASE_URL="http://localhost:8901" +TOKEN="PUT_YOUR_TOKEN_HERE" + +# Put one input into maintenance, with an optional remark +curl -s -X PUT "$BASE_URL/api/v1/maintenance/provider_a" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + --data-raw '{"reason":"Provider down"}' + +# Take every input out of maintenance (`*`, URL-encoded as %2A) +curl -s -X DELETE "$BASE_URL/api/v1/maintenance/%2A" -H "Authorization: Bearer $TOKEN" + +# Current state +curl -s "$BASE_URL/api/v1/maintenance" -H "Authorization: Bearer $TOKEN" +``` + +Every call answers with the complete state: + +```json +{ + "entries": { "provider_a": { "since": 1760350000, "reason": "Provider down" } }, + "inputs": ["provider_a", "provider_b"] +} +``` + +- `*` addresses all inputs known at the time of the call; an input literally named `*` cannot be addressed alone. +- Putting an input that is already in maintenance updates its remark and keeps `since`. +- Unknown inputs answer `404`, remarks longer than 500 characters `400`. +- Without Web UI authentication the endpoints are as open as the other `/api/v1` routes. + ## Available `/api/v1` Endpoints This is a compact operator-oriented overview of the `/api/v1` REST API groups currently registered by the backend. @@ -369,6 +421,14 @@ This is a compact operator-oriented overview of the `/api/v1` REST API groups cu | `GET` | `/api/v1/qos-snapshots/{stream_identity_key}` | QoS detail for one stream | | `GET` | `/api/v1/geoip/update` | Trigger GeoIP DB update | +### Maintenance mode + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/api/v1/maintenance` | Inputs in maintenance and all known input names (`maintenance.read`) | +| `PUT` | `/api/v1/maintenance/{input}` | Put an input (or `*` for all) into maintenance, optional `{"reason": "..."}` body (`maintenance.write`) | +| `DELETE` | `/api/v1/maintenance/{input}` | Take an input (or `*` for all) out of maintenance (`maintenance.write`) | + ### Recordings Recording REST is **commands only**. There is no list, task, status or quota @@ -659,6 +719,8 @@ With Web UI authentication enabled, many endpoints require matching permissions - `recording.create` - `recording.manage` - `recording.delete` +- `maintenance.read` +- `maintenance.write` If a request is rejected, verify the logged-in Web UI user's RBAC group assignments first. diff --git a/frontend/public/assets/i18n/ar.json b/frontend/public/assets/i18n/ar.json index 9a2a812aa..153ec6b4a 100644 --- a/frontend/public/assets/i18n/ar.json +++ b/frontend/public/assets/i18n/ar.json @@ -869,6 +869,10 @@ "SKIP_LIVE_DIRECT_SOURCE": "إذا تم التفعيل، لا يتم ربط التدفقات المباشرة مباشرة ولكن يتم تحويلها/حلها.", "SKIP_SERIES_DIRECT_SOURCE": "إذا تم التفعيل، لا يتم ربط حلقات المسلسلات مباشرة ولكن يتم تحويلها/حلها.", "SKIP_VIDEO_DIRECT_SOURCE": "إذا تم التفعيل، لا يتم ربط ملفات VOD مباشرة ولكن يتم تحويلها/حلها." + }, + "MAINTENANCE": { + "INPUT": "المدخل الذي يتم إيقافه عن الخدمة. أثناء الصيانة تحصل البثوث الجديدة على فيديو الصيانة ولا يتصل Tuliprox بمزوده؛ تستمر البثوث والتسجيلات الجارية.", + "REASON": "ملاحظة اختيارية تظهر في قائمة الصيانة، مثل سبب تعطل المزود. 500 حرف كحد أقصى." } }, "HINT": { @@ -942,6 +946,15 @@ "ADD_PROPERTY": "إضافة خاصية", "ADD_PROVIDER": "إضافة مزود", "ADD_QUALITY_TO_FILENAME": "الجودة", + "MAINTENANCE": "الصيانة", + "MAINTENANCE_ADD_ALL": "الكل إلى الصيانة", + "MAINTENANCE_COUNT": "{count} مدخلات قيد الصيانة", + "MAINTENANCE_COUNT_ONE": "مدخل واحد قيد الصيانة", + "MAINTENANCE_NONE": "لا يوجد مدخل قيد الصيانة", + "MAINTENANCE_REASON": "ملاحظة", + "MAINTENANCE_REMOVE_ALL": "إنهاء كل الصيانة", + "MAINTENANCE_SINCE": "منذ", + "MANAGE": "إدارة", "USE_METADATA": "استخدام البيانات الوصفية", "ADD_SERVER": "إضافة خادم", "ADD_STRIP_ENTRY": "إضافة إدخال تجريد", @@ -1286,6 +1299,7 @@ "MPEG_TS": "MPEG-TS", "MSG_KIND_ERROR": "خطأ", "MSG_KIND_INFO": "معلومات", + "MSG_KIND_MAINTENANCE_CHANGED": "تغيير وضع صيانة المدخل", "MSG_KIND_LIBRARY_SCAN_COMPLETED": "اكتمل فحص المكتبة", "MSG_KIND_LIBRARY_SCAN_FAILED": "فشل فحص المكتبة", "MSG_KIND_METADATA_UPDATE_COMPLETED": "اكتمل تحديث البيانات الوصفية", @@ -1941,6 +1955,11 @@ "MESSAGE": "أدخل بيانات اعتمادك", "FAILED": "فشل تسجيل الدخول" }, + "MAINTENANCE": { + "LOAD_FAILED": "تعذر تحميل حالة الصيانة", + "REASON_TOO_LONG": "يمكن أن تحتوي الملاحظة على 500 حرف كحد أقصى", + "SAVE_FAILED": "تعذر حفظ حالة الصيانة" + }, "SESSION": { "EXPIRED": "انتهت جلستك. يرجى تسجيل الدخول مرة أخرى." }, diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index abe10f21c..9210a59a3 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -869,6 +869,10 @@ "SKIP_LIVE_DIRECT_SOURCE": "If enabled, live streams are not linked directly but proxied/resolved.", "SKIP_SERIES_DIRECT_SOURCE": "If enabled, series episodes are not linked directly but proxied/resolved.", "SKIP_VIDEO_DIRECT_SOURCE": "If enabled, VOD files are not linked directly but proxied/resolved." + }, + "MAINTENANCE": { + "INPUT": "Input to take out of service. While it is in maintenance, new streams get the maintenance video and Tuliprox does not contact its provider; running streams and recordings continue.", + "REASON": "Optional remark shown in the maintenance list, for example why the provider is down. At most 500 characters." } }, "HINT": { @@ -940,6 +944,15 @@ "ADD_PROPERTY": "Add Property", "ADD_PROVIDER": "Add Provider", "ADD_QUALITY_TO_FILENAME": "Quality", + "MAINTENANCE": "Maintenance", + "MAINTENANCE_ADD_ALL": "All into maintenance", + "MAINTENANCE_COUNT": "{count} inputs in maintenance", + "MAINTENANCE_COUNT_ONE": "1 input in maintenance", + "MAINTENANCE_NONE": "No input in maintenance", + "MAINTENANCE_REASON": "Remark", + "MAINTENANCE_REMOVE_ALL": "End all maintenance", + "MAINTENANCE_SINCE": "Since", + "MANAGE": "Manage", "USE_METADATA": "Use metadata", "ADD_SERVER": "Add Server", "ADD_PLAN": "Add Plan", @@ -1287,6 +1300,7 @@ "MPEG_TS": "MPEG-TS", "MSG_KIND_ERROR": "Error", "MSG_KIND_INFO": "Info", + "MSG_KIND_MAINTENANCE_CHANGED": "Maintenance Changed", "MSG_KIND_LIBRARY_SCAN_COMPLETED": "Library Scan Completed", "MSG_KIND_LIBRARY_SCAN_FAILED": "Library Scan Failed", "MSG_KIND_METADATA_UPDATE_COMPLETED": "Metadata Update Completed", @@ -1941,6 +1955,11 @@ "MESSAGE": "Enter Your Credentials", "FAILED": "Failed to login" }, + "MAINTENANCE": { + "LOAD_FAILED": "Could not load maintenance state", + "REASON_TOO_LONG": "The remark may have at most 500 characters", + "SAVE_FAILED": "Could not save maintenance state" + }, "SESSION": { "EXPIRED": "Your session has expired. Please log in again." }, diff --git a/frontend/public/assets/i18n/es.json b/frontend/public/assets/i18n/es.json index 6354039d6..b54560abe 100644 --- a/frontend/public/assets/i18n/es.json +++ b/frontend/public/assets/i18n/es.json @@ -869,6 +869,10 @@ "SKIP_LIVE_DIRECT_SOURCE": "Si está habilitado, los streams en directo no se enlazan directamente sino que se proxyfican/resuelven.", "SKIP_SERIES_DIRECT_SOURCE": "Si está habilitado, los episodios de series no se enlazan directamente sino que se proxyfican/resuelven.", "SKIP_VIDEO_DIRECT_SOURCE": "Si está habilitado, los archivos VOD no se enlazan directamente sino que se proxyfican/resuelven." + }, + "MAINTENANCE": { + "INPUT": "Entrada que se retira del servicio. Mientras está en mantenimiento, las nuevas reproducciones reciben el vídeo de mantenimiento y Tuliprox no contacta con su proveedor; las reproducciones y grabaciones en curso continúan.", + "REASON": "Observación opcional que se muestra en la lista de mantenimiento, por ejemplo por qué el proveedor no está disponible. Máximo 500 caracteres." } }, "HINT": { @@ -940,6 +944,15 @@ "ADD_PROPERTY": "Añadir propiedad", "ADD_PROVIDER": "Añadir proveedor", "ADD_QUALITY_TO_FILENAME": "Calidad", + "MAINTENANCE": "Mantenimiento", + "MAINTENANCE_ADD_ALL": "Todas a mantenimiento", + "MAINTENANCE_COUNT": "{count} entradas en mantenimiento", + "MAINTENANCE_COUNT_ONE": "1 entrada en mantenimiento", + "MAINTENANCE_NONE": "Ninguna entrada en mantenimiento", + "MAINTENANCE_REASON": "Observación", + "MAINTENANCE_REMOVE_ALL": "Terminar todo el mantenimiento", + "MAINTENANCE_SINCE": "Desde", + "MANAGE": "Gestionar", "USE_METADATA": "Usar metadatos", "ADD_SERVER": "Añadir servidor", "ADD_PLAN": "Añadir plan", @@ -1287,6 +1300,7 @@ "MPEG_TS": "MPEG-TS", "MSG_KIND_ERROR": "Error", "MSG_KIND_INFO": "Info", + "MSG_KIND_MAINTENANCE_CHANGED": "Mantenimiento de entrada cambiado", "MSG_KIND_LIBRARY_SCAN_COMPLETED": "Escaneo de biblioteca completado", "MSG_KIND_LIBRARY_SCAN_FAILED": "Escaneo de biblioteca fallido", "MSG_KIND_METADATA_UPDATE_COMPLETED": "Actualización de metadatos completada", @@ -1941,6 +1955,11 @@ "MESSAGE": "Introduzca sus credenciales", "FAILED": "No se pudo iniciar sesión" }, + "MAINTENANCE": { + "LOAD_FAILED": "No se pudo cargar el estado de mantenimiento", + "REASON_TOO_LONG": "La observación puede tener como máximo 500 caracteres", + "SAVE_FAILED": "No se pudo guardar el estado de mantenimiento" + }, "SESSION": { "EXPIRED": "Su sesión ha expirado. Por favor, inicie sesión de nuevo." }, diff --git a/frontend/public/assets/i18n/ru.json b/frontend/public/assets/i18n/ru.json index bf64455a1..bb864e3f2 100644 --- a/frontend/public/assets/i18n/ru.json +++ b/frontend/public/assets/i18n/ru.json @@ -869,6 +869,10 @@ "SKIP_LIVE_DIRECT_SOURCE": "Если включено, прямые трансляции не связаны напрямую, а проксируются/разрешаются.", "SKIP_SERIES_DIRECT_SOURCE": "Если включено, эпизоды сериалов не связаны напрямую, а проксируются/разрешаются.", "SKIP_VIDEO_DIRECT_SOURCE": "Если включено, VOD-файлы не связаны напрямую, а проксируются/разрешаются." + }, + "MAINTENANCE": { + "INPUT": "Вход, который выводится из эксплуатации. Пока он на обслуживании, новые потоки получают видео обслуживания, а Tuliprox не обращается к его провайдеру; текущие потоки и записи продолжаются.", + "REASON": "Необязательное примечание в списке обслуживания, например почему провайдер недоступен. Не более 500 символов." } }, "HINT": { @@ -942,6 +946,15 @@ "ADD_PROPERTY": "Добавить свойство", "ADD_PROVIDER": "Добавить провайдера", "ADD_QUALITY_TO_FILENAME": "Качество", + "MAINTENANCE": "Обслуживание", + "MAINTENANCE_ADD_ALL": "Все на обслуживание", + "MAINTENANCE_COUNT": "Входов на обслуживании: {count}", + "MAINTENANCE_COUNT_ONE": "1 вход на обслуживании", + "MAINTENANCE_NONE": "Нет входов на обслуживании", + "MAINTENANCE_REASON": "Примечание", + "MAINTENANCE_REMOVE_ALL": "Завершить всё обслуживание", + "MAINTENANCE_SINCE": "С", + "MANAGE": "Управлять", "USE_METADATA": "Использовать метаданные", "ADD_SERVER": "Добавить сервер", "ADD_STRIP_ENTRY": "Добавить запись для удаления", @@ -1286,6 +1299,7 @@ "MPEG_TS": "MPEG-TS", "MSG_KIND_ERROR": "Ошибка", "MSG_KIND_INFO": "Информация", + "MSG_KIND_MAINTENANCE_CHANGED": "Изменён режим обслуживания входа", "MSG_KIND_LIBRARY_SCAN_COMPLETED": "Сканирование библиотеки завершено", "MSG_KIND_LIBRARY_SCAN_FAILED": "Ошибка сканирования библиотеки", "MSG_KIND_METADATA_UPDATE_COMPLETED": "Обновление метаданных завершено", @@ -1941,6 +1955,11 @@ "MESSAGE": "Введите свои учетные данные", "FAILED": "Не удалось войти" }, + "MAINTENANCE": { + "LOAD_FAILED": "Не удалось загрузить состояние обслуживания", + "REASON_TOO_LONG": "Примечание может содержать не более 500 символов", + "SAVE_FAILED": "Не удалось сохранить состояние обслуживания" + }, "SESSION": { "EXPIRED": "Ваша сессия истекла. Войдите снова." }, diff --git a/frontend/scss/app/_component.scss b/frontend/scss/app/_component.scss index 25ca8f177..abff837d5 100644 --- a/frontend/scss/app/_component.scss +++ b/frontend/scss/app/_component.scss @@ -17,6 +17,7 @@ @forward "components/preferences"; @forward "components/dashboard/action_card"; @forward "components/dashboard/credits_action_card"; +@forward "components/dashboard/maintenance"; @forward "components/dashboard/status_card"; @forward "components/dashboard/sparkline"; @forward "components/dashboard/dashboard_view"; diff --git a/frontend/scss/app/components/dashboard/_maintenance.scss b/frontend/scss/app/components/dashboard/_maintenance.scss new file mode 100644 index 000000000..48b991d5a --- /dev/null +++ b/frontend/scss/app/components/dashboard/_maintenance.scss @@ -0,0 +1,121 @@ +.tp__card--maintenance { + background-color: color-mix(in srgb, var(--theme-brand-danger) 22%, transparent); + border-color: var(--theme-brand-danger-strong); +} + +// Doubled class: `.tp__custom-dialog` sets `width: auto` and `max-width: 90%` and must not win. +.tp__custom-dialog.tp__maintenance-dialog { + width: min(64rem, 94vw); + max-width: 94vw; + height: min(44rem, 86vh); +} + +.tp__maintenance-dialog { + display: flex; + flex-flow: column; + gap: var(--gap-default); + padding: var(--padding-default); + box-sizing: border-box; + + h2 { + margin: 0; + } + + // Bordered panel like the watch form of a target: input, remark and the add button in one row. + &__form { + display: flex; + flex-flow: row nowrap; + align-items: flex-end; + gap: var(--gap-default); + flex-shrink: 0; + padding: var(--padding-default); + border: 1px solid var(--border-color); + border-radius: var(--border-radius); + + > .tp__form-field { + flex: 1 1 0; + min-width: 0; + } + + > .tp__input { + flex: 2 1 0; + min-width: 0; + } + } + + // Same height as the input boxes next to it, so it lines up with them and not with the labels. + &__add.tp__icon-button { + flex: 0 0 auto; + height: 2.5rem; + width: 2.5rem; + + .svg-icon { + height: 1.75rem; + width: 1.75rem; + } + } + + // Many inputs in maintenance scroll here; title, form and toolbar stay in place. + &__list { + flex: 1 1 auto; + min-height: 8rem; + overflow-y: auto; + border: 1px solid var(--border-color); + border-radius: var(--border-radius); + } + + &__empty { + margin: 0; + padding: var(--padding-default); + color: var(--modest-text-color); + } + + &__table { + width: 100%; + border-collapse: collapse; + + th, + td { + text-align: start; + padding: var(--padding-small) var(--padding-default); + border-bottom: 1px solid var(--border-color); + } + + thead th { + position: sticky; + top: 0; + z-index: 1; + background-color: var(--card-background-color); + color: var(--modest-text-color); + } + + tbody tr:last-child td { + border-bottom: none; + } + } + + // Draft rows that Save has not sent yet. + &__row--unsaved td { + font-style: italic; + } + + &__actions { + display: flex; + gap: var(--gap-small); + justify-content: flex-end; + } + + .tp__dialog__toolbar { + flex-shrink: 0; + } +} + +@media (max-width: 860px) { + .tp__maintenance-dialog__form { + flex-flow: row wrap; + + > .tp__form-field { + flex: 1 1 100%; + } + } +} diff --git a/frontend/src/app/components/dashboard/dashboard_view.rs b/frontend/src/app/components/dashboard/dashboard_view.rs index 47c829dfb..30e039944 100644 --- a/frontend/src/app/components/dashboard/dashboard_view.rs +++ b/frontend/src/app/components/dashboard/dashboard_view.rs @@ -2,18 +2,21 @@ use crate::{ app::{ components::{ Card, CreditsActionCard, DiscordActionCard, DocumentationActionCard, GithubActionCard, IpinfoActionCard, - UserActionCard, VersionActionCard, + MaintenanceActionCard, UserActionCard, VersionActionCard, }, context::StatusContext, }, + hooks::use_service_context, i18n::use_translation, }; +use shared::model::Permission; use yew::prelude::*; #[component] pub fn DashboardView() -> Html { let translate = use_translation(); let status_ctx = use_context::().expect("Status context not found"); + let can_see_maintenance = use_service_context().auth.has_permission(Permission::MaintenanceRead); html! {
@@ -32,6 +35,7 @@ pub fn DashboardView() -> Html { + if can_see_maintenance { }
diff --git a/frontend/src/app/components/dashboard/maintenance_action_card.rs b/frontend/src/app/components/dashboard/maintenance_action_card.rs new file mode 100644 index 000000000..24a023552 --- /dev/null +++ b/frontend/src/app/components/dashboard/maintenance_action_card.rs @@ -0,0 +1,158 @@ +use super::{ + maintenance_dialog::MaintenanceDialog, + maintenance_state::{is_alert, merge_loaded, preview_names}, +}; +use crate::{ + app::{ + components::{ActionCard, TextButton}, + CardContext, + }, + hooks::use_service_context, + i18n::use_translation, + model::EventMessage, +}; +use shared::model::{MaintenanceStateDto, Permission}; +use std::rc::Rc; +use yew::prelude::*; + +const ALERT_CLASS: &str = "tp__card--maintenance"; + +fn dialog_portal(content: Html) -> Html { + match web_sys::window().and_then(|window| window.document()).and_then(|document| document.body()) { + Some(body) => yew::create_portal(content, body.into()), + None => content, + } +} + +#[component] +pub fn MaintenanceActionCard() -> Html { + let services = use_service_context(); + let translate = use_translation(); + let card_ctx = use_context::(); + let state = use_state(|| Rc::new(MaintenanceStateDto::default())); + // The latest state, readable from the subscription and from a load in flight without going + // through a stale state handle. + let latest = use_mut_ref(|| Rc::new(MaintenanceStateDto::default())); + // Counts change events; a load that saw a newer event than the one it started after is stale. + let event_generation = use_mut_ref(|| 0_u64); + let dialog_open = use_state(|| false); + let can_read = services.auth.has_permission(Permission::MaintenanceRead); + let can_write = services.auth.has_permission(Permission::MaintenanceWrite); + + let apply_state = { + let state = state.clone(); + let latest = latest.clone(); + Callback::from(move |next: MaintenanceStateDto| { + let next = Rc::new(next); + *latest.borrow_mut() = Rc::clone(&next); + state.set(next); + }) + }; + + let reload = { + let services = services.clone(); + let translate = translate.clone(); + let apply_state = apply_state.clone(); + let latest = latest.clone(); + let event_generation = event_generation.clone(); + Callback::from(move |()| { + let services = services.clone(); + let translate = translate.clone(); + let apply_state = apply_state.clone(); + let latest = latest.clone(); + let event_generation = event_generation.clone(); + let started_at = *event_generation.borrow(); + wasm_bindgen_futures::spawn_local(async move { + match services.maintenance.get().await { + Ok(loaded) => { + let stale = *event_generation.borrow() != started_at; + let current = Rc::clone(&latest.borrow()); + apply_state.emit(merge_loaded(¤t, loaded, stale)); + } + Err(_) => services.toastr.error(translate.t("MESSAGES.MAINTENANCE.LOAD_FAILED")), + } + }); + }) + }; + + { + let services = services.clone(); + let apply_state = apply_state.clone(); + let latest = latest.clone(); + let event_generation = event_generation.clone(); + let reload = reload.clone(); + use_effect_with(can_read, move |&can_read| { + let subscription = can_read.then(|| { + reload.emit(()); + services.event.subscribe(move |message| match message { + EventMessage::MaintenanceChanged(entries) => { + *event_generation.borrow_mut() += 1; + let inputs = latest.borrow().inputs.clone(); + apply_state.emit(MaintenanceStateDto { entries: (*entries).clone(), inputs }); + } + // Events sent while the socket was down are lost, and a source reload can + // add inputs: read the state again. + EventMessage::WebSocketStatus(true) | EventMessage::ConfigChange(_) => reload.emit(()), + _ => {} + }) + }); + move || { + if let Some(subscription) = subscription { + services.event.unsubscribe(subscription); + } + } + }); + } + + { + let alert = is_alert(&state); + use_effect_with(alert, move |&alert| { + if let Some(card_ctx) = card_ctx { + card_ctx.custom_class.set(if alert { ALERT_CLASS.to_string() } else { String::new() }); + } + || () + }); + } + + if !can_read { + return Html::default(); + } + + let subtitle = if is_alert(&state) { + format!( + "{}: {}", + if state.entries.len() == 1 { + translate.t("LABEL.MAINTENANCE_COUNT_ONE") + } else { + translate.t("LABEL.MAINTENANCE_COUNT").replace("{count}", &state.entries.len().to_string()) + }, + preview_names(&state, 3) + ) + } else { + translate.t("LABEL.MAINTENANCE_NONE") + }; + + let open_dialog = { + let dialog_open = dialog_open.clone(); + Callback::from(move |_: String| dialog_open.set(true)) + }; + let close_dialog = { + let dialog_open = dialog_open.clone(); + Callback::from(move |()| dialog_open.set(false)) + }; + + html! { + <> + + + + if *dialog_open { + // Rendered into : inside the card the backdrop would share the card's stacking + // context and let pointer events reach the dashboard behind it. + { dialog_portal(html! { + + }) } + } + + } +} diff --git a/frontend/src/app/components/dashboard/maintenance_dialog.rs b/frontend/src/app/components/dashboard/maintenance_dialog.rs new file mode 100644 index 000000000..a6ca25787 --- /dev/null +++ b/frontend/src/app/components/dashboard/maintenance_dialog.rs @@ -0,0 +1,258 @@ +use super::maintenance_state::{ + add_all_to_draft, add_to_draft, pending_changes, reason_too_long, relative_duration, PendingChanges, +}; +use crate::{ + app::components::{ + input::Input, selection_first_owned, CustomDialog, DropDownOption, DropDownSelection, IconButton, Select, + TextButton, + }, + config_field_child, + hooks::use_service_context, + i18n::use_translation, + services::MaintenanceService, +}; +use shared::model::{MaintenanceEntries, MaintenanceStateDto}; +use std::rc::Rc; +use wasm_bindgen::JsValue; +use yew::prelude::*; + +#[derive(Properties, Clone, PartialEq)] +pub struct MaintenanceDialogProps { + pub state: Rc, + pub can_write: bool, + pub on_state: Callback, + pub on_close: Callback<()>, +} + +#[allow(clippy::cast_precision_loss)] +fn local_time(since: i64) -> String { + let millis = since.saturating_mul(1000) as f64; + String::from(js_sys::Date::new(&JsValue::from_f64(millis)).to_locale_string("default", &JsValue::UNDEFINED)) +} + +#[allow(clippy::cast_possible_truncation)] +fn now_secs() -> i64 { (js_sys::Date::now() / 1000.0) as i64 } + +/// Sends the pending changes one by one and reads the resulting state back. +async fn apply_changes( + service: &MaintenanceService, + changes: PendingChanges, +) -> Result { + for name in &changes.deletes { + service.remove(name).await?; + } + for (name, reason) in changes.puts { + service.add(&name, reason).await?; + } + service.get().await +} + +/// Edits a draft of the inputs in maintenance. Adding, removing and the bulk actions only change +/// the draft; Save sends the difference to the server, Close discards it. +#[component] +pub fn MaintenanceDialog(props: &MaintenanceDialogProps) -> Html { + let services = use_service_context(); + let translate = use_translation(); + let draft = { + let entries = props.state.entries.clone(); + use_state(move || entries) + }; + let selected = use_state(|| None::); + let reason = use_state(String::new); + let pending = use_state(|| false); + + let too_long = reason_too_long(&reason); + let changes = pending_changes(&props.state.entries, &draft); + + let on_add = { + let draft = draft.clone(); + let selected = selected.clone(); + let reason = reason.clone(); + Callback::from(move |(_, event): (String, MouseEvent)| { + event.prevent_default(); + let Some(name) = (*selected).clone() else { return }; + if reason_too_long(&reason) { + return; + } + draft.set(add_to_draft(&draft, &name, &reason, now_secs())); + selected.set(None); + reason.set(String::new()); + }) + }; + + let on_add_all = { + let draft = draft.clone(); + let reason = reason.clone(); + let inputs = props.state.inputs.clone(); + Callback::from(move |_: String| { + if !reason_too_long(&reason) { + draft.set(add_all_to_draft(&draft, &inputs, &reason, now_secs())); + } + }) + }; + + let on_remove_all = { + let draft = draft.clone(); + Callback::from(move |_: String| draft.set(MaintenanceEntries::new())) + }; + + let on_save = { + let services = services.clone(); + let translate = translate.clone(); + let on_state = props.on_state.clone(); + let on_close = props.on_close.clone(); + let pending = pending.clone(); + let saved = props.state.entries.clone(); + let draft = draft.clone(); + Callback::from(move |_: String| { + let changes = pending_changes(&saved, &draft); + if changes.is_empty() { + on_close.emit(()); + return; + } + let services = services.clone(); + let translate = translate.clone(); + let on_state = on_state.clone(); + let on_close = on_close.clone(); + let pending = pending.clone(); + pending.set(true); + wasm_bindgen_futures::spawn_local(async move { + match apply_changes(&services.maintenance, changes).await { + Ok(next) => { + on_state.emit(next); + on_close.emit(()); + } + // A partial save is visible through the change event; the draft stays for a retry. + Err(_) => services.toastr.error(translate.t("MESSAGES.MAINTENANCE.SAVE_FAILED")), + } + pending.set(false); + }); + }) + }; + + let now = now_secs(); + let rows = draft.iter().map(|(name, entry)| { + let unsaved = props.state.entries.get(name) != Some(entry); + let on_remove = { + let draft = draft.clone(); + let name = name.clone(); + Callback::from(move |_: String| { + let mut next = (*draft).clone(); + next.remove(&name); + draft.set(next); + }) + }; + let on_edit = { + let selected = selected.clone(); + let reason = reason.clone(); + let name = name.clone(); + let remark = entry.reason.clone().unwrap_or_default(); + Callback::from(move |_: String| { + selected.set(Some(name.clone())); + reason.set(remark.clone()); + }) + }; + html! { + + {name} + {relative_duration(entry.since, now)} + {entry.reason.clone().unwrap_or_default()} + if props.can_write { + + + + + } + + } + }); + + // Inputs not in the draft yet, plus the one being edited. + let options = { + let mut names: Vec<&str> = + props.state.inputs.iter().map(String::as_str).filter(|name| !draft.contains_key(*name)).collect(); + if let Some(editing) = selected.as_deref() { + if !names.contains(&editing) { + names.insert(0, editing); + } + } + Rc::new( + names + .into_iter() + .map(|name| DropDownOption::new(name, html! { name }, selected.as_deref() == Some(name))) + .collect::>(), + ) + }; + let on_select = { + let selected = selected.clone(); + Callback::from(move |(_, selection): (String, DropDownSelection)| { + selected.set(selection_first_owned(selection)); + }) + }; + let on_reason = { + let reason = reason.clone(); + Callback::from(move |value: String| reason.set(value)) + }; + let close = props.on_close.clone(); + let close_button = { + let close = props.on_close.clone(); + Callback::from(move |_: String| close.emit(())) + }; + let busy = *pending; + + html! { + +

{translate.t("LABEL.MAINTENANCE")}

+ if props.can_write { +
+ { config_field_child!(translate.t("LABEL.INPUT"), "MAINTENANCE.INPUT", { + html! { + + +
+ } +
+ if draft.is_empty() { +

{translate.t("LABEL.MAINTENANCE_NONE")}

+ } else { + + + + + + if props.can_write { } + + {for rows} +
{translate.t("LABEL.INPUT")}{translate.t("LABEL.MAINTENANCE_SINCE")}{translate.t("LABEL.MAINTENANCE_REASON")}
+ } +
+
+
+ if props.can_write { + + + } +
+
+ + if props.can_write { + + } +
+
+ + } +} diff --git a/frontend/src/app/components/dashboard/maintenance_state.rs b/frontend/src/app/components/dashboard/maintenance_state.rs new file mode 100644 index 000000000..8a24f9a6b --- /dev/null +++ b/frontend/src/app/components/dashboard/maintenance_state.rs @@ -0,0 +1,183 @@ +//! What the maintenance card and dialog derive from the server state. + +use shared::model::{InputMaintenance, MaintenanceEntries, MaintenanceStateDto, MAINTENANCE_REASON_MAX_CHARS}; + +pub fn is_alert(state: &MaintenanceStateDto) -> bool { !state.entries.is_empty() } + +/// The first `max` input names, with an ellipsis when there are more. +pub fn preview_names(state: &MaintenanceStateDto, max: usize) -> String { + let mut names: Vec<&str> = state.entries.keys().map(String::as_str).take(max).collect(); + if state.entries.len() > max { + names.push("…"); + } + names.join(", ") +} + +/// Applies a loaded state. A load that raced with a newer change event keeps the event's +/// entries and only takes the input list. +pub fn merge_loaded(current: &MaintenanceStateDto, loaded: MaintenanceStateDto, stale: bool) -> MaintenanceStateDto { + if stale { + MaintenanceStateDto { entries: current.entries.clone(), inputs: loaded.inputs } + } else { + loaded + } +} + +/// The server rejects longer remarks; the dialog says so before sending. +pub fn reason_too_long(reason: &str) -> bool { reason.trim().chars().count() > MAINTENANCE_REASON_MAX_CHARS } + +fn normalized(reason: &str) -> Option { + let reason = reason.trim(); + (!reason.is_empty()).then(|| reason.to_string()) +} + +/// Puts `name` into the draft, or updates its remark; an existing entry keeps its `since`. +pub fn add_to_draft(draft: &MaintenanceEntries, name: &str, reason: &str, now: i64) -> MaintenanceEntries { + let mut next = draft.clone(); + let reason = normalized(reason); + next.entry(name.to_string()) + .and_modify(|entry| entry.reason.clone_from(&reason)) + .or_insert(InputMaintenance { since: now, reason }); + next +} + +/// Puts every known input that is not in the draft yet into it. +pub fn add_all_to_draft(draft: &MaintenanceEntries, inputs: &[String], reason: &str, now: i64) -> MaintenanceEntries { + let mut next = draft.clone(); + let reason = normalized(reason); + for name in inputs { + next.entry(name.clone()).or_insert_with(|| InputMaintenance { since: now, reason: reason.clone() }); + } + next +} + +/// What Save has to send to turn the saved state into the draft. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct PendingChanges { + /// New inputs and changed remarks. + pub puts: Vec<(String, Option)>, + pub deletes: Vec, +} + +impl PendingChanges { + pub fn is_empty(&self) -> bool { self.puts.is_empty() && self.deletes.is_empty() } +} + +pub fn pending_changes(saved: &MaintenanceEntries, draft: &MaintenanceEntries) -> PendingChanges { + let puts = draft + .iter() + .filter(|(name, entry)| saved.get(*name).is_none_or(|current| current.reason != entry.reason)) + .map(|(name, entry)| (name.clone(), entry.reason.clone())) + .collect(); + let deletes = saved.keys().filter(|name| !draft.contains_key(*name)).cloned().collect(); + PendingChanges { puts, deletes } +} + +/// How long an input has been in maintenance, in the two largest units. +pub fn relative_duration(since: i64, now: i64) -> String { + let secs = (now - since).max(0); + let (days, hours, minutes) = (secs / 86_400, secs % 86_400 / 3600, secs % 3600 / 60); + match (days, hours, minutes) { + (0, 0, 0) => "<1m".to_string(), + (0, 0, m) => format!("{m}m"), + (0, h, m) => format!("{h}h {m}m"), + (d, h, _) => format!("{d}d {h}h"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use shared::model::{InputMaintenance, MaintenanceEntries, MaintenanceStateDto}; + + fn state(in_maintenance: &[&str], inputs: &[&str]) -> MaintenanceStateDto { + MaintenanceStateDto { + entries: in_maintenance + .iter() + .map(|name| ((*name).to_string(), InputMaintenance { since: 1, reason: None })) + .collect(), + inputs: inputs.iter().map(ToString::to_string).collect(), + } + } + + #[test] + fn card_is_alert_only_with_entries() { + assert!(!is_alert(&state(&[], &["a"]))); + assert!(is_alert(&state(&["a"], &["a"]))); + } + + #[test] + fn preview_names_are_truncated() { + let s = state(&["a", "b", "c", "d"], &["a", "b", "c", "d"]); + assert_eq!(preview_names(&s, 3), "a, b, c, …"); + assert_eq!(preview_names(&state(&["a"], &["a"]), 3), "a"); + } + + #[test] + fn a_stale_load_keeps_newer_entries_but_takes_the_input_list() { + let current = state(&["a"], &["a"]); + let loaded = state(&[], &["a", "b"]); + let merged = merge_loaded(¤t, loaded.clone(), true); + assert_eq!(merged.entries, current.entries, "entries from a newer event win"); + assert_eq!(merged.inputs, loaded.inputs); + assert_eq!(merge_loaded(¤t, loaded.clone(), false), loaded); + } + + #[test] + fn reason_limit_is_checked_in_characters() { + let limit = shared::model::MAINTENANCE_REASON_MAX_CHARS; + assert!(!reason_too_long(&"ä".repeat(limit))); + assert!(reason_too_long(&"x".repeat(limit + 1))); + assert!(!reason_too_long(&format!(" {} ", "x".repeat(limit))), "surrounding whitespace is trimmed"); + } + + fn entries(items: &[(&str, i64, Option<&str>)]) -> MaintenanceEntries { + items + .iter() + .map(|(name, since, reason)| { + ((*name).to_string(), InputMaintenance { since: *since, reason: reason.map(ToString::to_string) }) + }) + .collect() + } + + #[test] + fn adding_to_the_draft_keeps_since_of_an_existing_entry() { + let draft = entries(&[("a", 10, Some("old"))]); + let updated = add_to_draft(&draft, "a", " new ", 99); + assert_eq!(updated, entries(&[("a", 10, Some("new"))])); + let added = add_to_draft(&draft, "b", " ", 99); + assert_eq!(added, entries(&[("a", 10, Some("old")), ("b", 99, None)])); + } + + #[test] + fn pending_changes_lists_puts_for_new_or_changed_and_deletes_for_removed() { + let saved = entries(&[("a", 1, None), ("b", 1, Some("x")), ("c", 1, None)]); + let draft = entries(&[("a", 1, None), ("b", 5, Some("y")), ("d", 9, Some("z"))]); + let changes = pending_changes(&saved, &draft); + assert_eq!( + changes.puts, + vec![("b".to_string(), Some("y".to_string())), ("d".to_string(), Some("z".to_string()))] + ); + assert_eq!(changes.deletes, vec!["c".to_string()]); + assert!(!changes.is_empty()); + assert!(pending_changes(&saved, &saved).is_empty()); + } + + #[test] + fn draft_bulk_helpers_cover_all_inputs() { + let s = state(&["a"], &["a", "b"]); + let all = add_all_to_draft(&s.entries, &s.inputs, "outage", 7); + assert_eq!(all.len(), 2); + assert_eq!(all["a"].since, 1, "existing entries keep their since"); + assert_eq!(all["b"], InputMaintenance { since: 7, reason: Some("outage".into()) }); + } + + #[test] + fn relative_duration_is_compact() { + assert_eq!(relative_duration(1_000, 1_000 + 59), "<1m"); + assert_eq!(relative_duration(1_000, 1_000 + 300), "5m"); + assert_eq!(relative_duration(1_000, 1_000 + 3 * 3600 + 120), "3h 2m"); + assert_eq!(relative_duration(1_000, 1_000 + 2 * 86_400 + 3600), "2d 1h"); + assert_eq!(relative_duration(1_000, 500), "<1m", "clock skew never shows a negative duration"); + } +} diff --git a/frontend/src/app/components/dashboard/mod.rs b/frontend/src/app/components/dashboard/mod.rs index ed695f5f1..f336152ef 100644 --- a/frontend/src/app/components/dashboard/mod.rs +++ b/frontend/src/app/components/dashboard/mod.rs @@ -6,6 +6,9 @@ mod documentation_action_card; mod github_action_card; mod ipinfo_action_card; mod log_console; +mod maintenance_action_card; +mod maintenance_dialog; +mod maintenance_state; mod metrics_history; mod playlist_progress_status_card; mod sparkline; @@ -20,7 +23,7 @@ mod streams_view; pub use self::{ action_card::*, credits_action_card::*, dashboard_view::*, discord_action_card::*, documentation_action_card::*, - github_action_card::*, ipinfo_action_card::*, log_console::*, metrics_history::*, playlist_progress_status_card::*, - sparkline::*, stats_view::*, status_card::*, stream_display::*, stream_history_view::*, streams_view::*, - user_action_card::*, version_action_card::*, + github_action_card::*, ipinfo_action_card::*, log_console::*, maintenance_action_card::*, metrics_history::*, + playlist_progress_status_card::*, sparkline::*, stats_view::*, status_card::*, stream_display::*, + stream_history_view::*, streams_view::*, user_action_card::*, version_action_card::*, }; diff --git a/frontend/src/hooks/use_service_context.rs b/frontend/src/hooks/use_service_context.rs index 6dc280031..31e85ba03 100644 --- a/frontend/src/hooks/use_service_context.rs +++ b/frontend/src/hooks/use_service_context.rs @@ -23,6 +23,7 @@ pub struct Services { pub websocket: Rc, pub rbac: Rc, pub flags: Rc, + pub maintenance: Rc, } impl Services { @@ -41,6 +42,7 @@ impl Services { let rbac = Rc::new(RbacService::new()); let flags = Rc::new(flags_service); let user_settings = Rc::new(crate::services::UserSettingsService::new()); + let maintenance = Rc::new(crate::services::MaintenanceService::new()); Self { auth, user_settings, @@ -56,6 +58,7 @@ impl Services { websocket, rbac, flags, + maintenance, } } } diff --git a/frontend/src/model/event_message.rs b/frontend/src/model/event_message.rs index f2bc9d30a..86b23d2e9 100644 --- a/frontend/src/model/event_message.rs +++ b/frontend/src/model/event_message.rs @@ -29,6 +29,7 @@ pub enum EventMessage { tasks: Rc>, }, RecordingRulesChanged, + MaintenanceChanged(Rc), /// The socket refused to serve recordings for an actionable reason. /// Carries the same stable code the REST routes use, so views map /// both through one i18n table. diff --git a/frontend/src/services/maintenance_service.rs b/frontend/src/services/maintenance_service.rs new file mode 100644 index 000000000..6fc3eedeb --- /dev/null +++ b/frontend/src/services/maintenance_service.rs @@ -0,0 +1,34 @@ +use crate::{ + error::Error, + services::{get_base_href, request_delete, request_get, request_put}, +}; +use shared::{ + model::{MaintenanceRequestDto, MaintenanceStateDto}, + utils::concat_path_leading_slash, +}; + +pub struct MaintenanceService { + path: String, +} +impl Default for MaintenanceService { + fn default() -> Self { Self::new() } +} +impl MaintenanceService { + pub fn new() -> Self { Self { path: concat_path_leading_slash(&get_base_href(), "api/v1/maintenance") } } + fn input_path(&self, input: &str) -> String { + format!("{}/{}", self.path, String::from(js_sys::encode_uri_component(input))) + } + pub async fn get(&self) -> Result { + request_get(&self.path, None, None).await?.ok_or(Error::DeserializeError) + } + /// `input` may be `*` for all inputs. Re-adding an input updates its remark. + pub async fn add(&self, input: &str, reason: Option) -> Result { + request_put(&self.input_path(input), MaintenanceRequestDto { reason }, None, None) + .await? + .ok_or(Error::DeserializeError) + } + /// `input` may be `*` for all inputs. + pub async fn remove(&self, input: &str) -> Result { + request_delete(&self.input_path(input), None, None).await?.ok_or(Error::DeserializeError) + } +} diff --git a/frontend/src/services/mod.rs b/frontend/src/services/mod.rs index 22736095f..e86aaf7d6 100644 --- a/frontend/src/services/mod.rs +++ b/frontend/src/services/mod.rs @@ -3,6 +3,7 @@ mod config_service; mod dialog_service; mod event_service; mod flags_service; +mod maintenance_service; mod playlist_service; mod rbac_service; mod recording_service; @@ -17,10 +18,10 @@ mod user_service; mod websocket_service; pub use self::{ - auth_service::*, config_service::*, dialog_service::*, event_service::*, flags_service::*, playlist_service::*, - rbac_service::*, recording_service::*, requests::*, status_service::*, stream_history_service::*, - streams_service::*, target_bouquet_service::*, toastr_service::*, user_api_service::*, user_service::*, - websocket_service::*, + auth_service::*, config_service::*, dialog_service::*, event_service::*, flags_service::*, maintenance_service::*, + playlist_service::*, rbac_service::*, recording_service::*, requests::*, status_service::*, + stream_history_service::*, streams_service::*, target_bouquet_service::*, toastr_service::*, user_api_service::*, + user_service::*, websocket_service::*, }; mod user_settings_service; diff --git a/frontend/src/services/websocket_service.rs b/frontend/src/services/websocket_service.rs index d978941cb..ac8f951e6 100644 --- a/frontend/src/services/websocket_service.rs +++ b/frontend/src/services/websocket_service.rs @@ -482,6 +482,9 @@ impl WebSocketService { ProtocolMessage::RecordingRulesChanged => { event_service.broadcast(EventMessage::RecordingRulesChanged); } + ProtocolMessage::MaintenanceChangedResponse(entries) => { + event_service.broadcast(EventMessage::MaintenanceChanged(Rc::new(entries))); + } ProtocolMessage::RecordingWsError { code } => { event_service.broadcast(EventMessage::RecordingUnavailable { code }); } diff --git a/resources/maintenance.jpg b/resources/maintenance.jpg new file mode 100644 index 000000000..b8cfed0e9 Binary files /dev/null and b/resources/maintenance.jpg differ diff --git a/shared/src/defaults/hls.rs b/shared/src/defaults/hls.rs index f6644ba88..e365f610c 100644 --- a/shared/src/defaults/hls.rs +++ b/shared/src/defaults/hls.rs @@ -47,5 +47,6 @@ pub const LOW_PRIORITY_PREEMPTED: &str = "low_priority_preempted.ts"; pub const USER_ACCOUNT_EXPIRED: &str = "user_account_expired.ts"; pub const PANEL_API_PROVISIONING: &str = "panel_api_provisioning.ts"; pub const HLS_SESSION_OR_LEASE_EXPIRED: &str = "hls_session_or_lease_expired.ts"; +pub const MAINTENANCE: &str = "maintenance.ts"; pub const PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT: usize = 6; pub const PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX: &str = "panel_api_provisioning_hls_"; diff --git a/shared/src/model/auth/permission.rs b/shared/src/model/auth/permission.rs index b7827d550..ef2838bb4 100644 --- a/shared/src/model/auth/permission.rs +++ b/shared/src/model/auth/permission.rs @@ -20,7 +20,9 @@ create_bitset!( RecordingRead, RecordingCreate, RecordingManage, - RecordingDelete + RecordingDelete, + MaintenanceRead, + MaintenanceWrite ); pub const PERM_ALL: PermissionSet = PermissionSet::ALL; @@ -44,6 +46,8 @@ pub const PERMISSION_NAMES: &[(&str, Permission)] = &[ ("recording.create", Permission::RecordingCreate), ("recording.manage", Permission::RecordingManage), ("recording.delete", Permission::RecordingDelete), + ("maintenance.read", Permission::MaintenanceRead), + ("maintenance.write", Permission::MaintenanceWrite), ]; pub fn permission_from_name(name: &str) -> Option { @@ -58,6 +62,15 @@ pub fn permission_to_name(perm: Permission) -> Option<&'static str> { mod tests { use super::*; + #[test] + fn maintenance_permissions_have_stable_names() { + assert_eq!(permission_from_name("maintenance.read"), Some(Permission::MaintenanceRead)); + assert_eq!(permission_from_name("maintenance.write"), Some(Permission::MaintenanceWrite)); + assert_eq!(permission_to_name(Permission::MaintenanceWrite), Some("maintenance.write")); + assert!(PERM_ALL.contains(Permission::MaintenanceRead)); + assert!(PERM_ALL.contains(Permission::MaintenanceWrite)); + } + #[test] fn test_permission_set_new_is_empty() { let set = PermissionSet::new(); @@ -173,13 +186,15 @@ mod tests { ("recording.create", 1 << 15), ("recording.manage", 1 << 16), ("recording.delete", 1 << 17), + ("maintenance.read", 1 << 18), + ("maintenance.write", 1 << 19), ]; // The table is the layout of this schema version. Changing a bit means // editing the table, and the version beside it has to move with it so // tokens minted under the old layout fail closed. assert_eq!( crate::model::CURRENT_PERMISSION_SCHEMA_VERSION, - 4, + 5, "the bit table changed: bump CURRENT_PERMISSION_SCHEMA_VERSION and this expected value together" ); assert_eq!(PermissionSet::VARIANT_COUNT, expected.len()); diff --git a/shared/src/model/auth/user.rs b/shared/src/model/auth/user.rs index 2efff9c73..f5daf6eb9 100644 --- a/shared/src/model/auth/user.rs +++ b/shared/src/model/auth/user.rs @@ -16,7 +16,7 @@ pub const TOKEN_NO_AUTH: &str = "authorized"; /// /// The layout this version stands for is pinned in the permission bit table /// test, next to the expected value of this constant. -pub const CURRENT_PERMISSION_SCHEMA_VERSION: u16 = 4; +pub const CURRENT_PERMISSION_SCHEMA_VERSION: u16 = 5; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Claims { diff --git a/shared/src/model/custom_video_stream_type.rs b/shared/src/model/custom_video_stream_type.rs index 7a34140ea..2c787a27a 100644 --- a/shared/src/model/custom_video_stream_type.rs +++ b/shared/src/model/custom_video_stream_type.rs @@ -17,6 +17,7 @@ pub enum CustomVideoStreamType { UserAccountExpired, Provisioning, HlsSessionOrLeaseExpired, + Maintenance, } impl fmt::Display for CustomVideoStreamType { @@ -29,6 +30,7 @@ impl fmt::Display for CustomVideoStreamType { CustomVideoStreamType::UserAccountExpired => "user_account_expired", CustomVideoStreamType::Provisioning => "provisioning", CustomVideoStreamType::HlsSessionOrLeaseExpired => "hls_session_or_lease_expired", + CustomVideoStreamType::Maintenance => "maintenance", }; write!(f, "{s}") } @@ -46,6 +48,7 @@ impl FromStr for CustomVideoStreamType { "user_account_expired" => Ok(Self::UserAccountExpired), "provisioning" => Ok(Self::Provisioning), "hls_session_or_lease_expired" => Ok(Self::HlsSessionOrLeaseExpired), + "maintenance" => Ok(Self::Maintenance), _ => Err(TuliproxError::Config(format!("Unknown stream type: {s}"))), } } @@ -68,3 +71,16 @@ impl<'de> Deserialize<'de> for CustomVideoStreamType { Self::from_str(&s).map_err(serde::de::Error::custom) } } + +#[cfg(test)] +mod tests { + use super::CustomVideoStreamType; + use std::str::FromStr; + + #[test] + fn maintenance_round_trips() { + let parsed = CustomVideoStreamType::from_str("maintenance").unwrap(); + assert_eq!(parsed, CustomVideoStreamType::Maintenance); + assert_eq!(parsed.to_string(), "maintenance"); + } +} diff --git a/shared/src/model/event/kind.rs b/shared/src/model/event/kind.rs index fc60c9c7e..519b284fc 100644 --- a/shared/src/model/event/kind.rs +++ b/shared/src/model/event/kind.rs @@ -60,6 +60,8 @@ pub enum EventKind { /// Payload-free progress nudge, appended last so no existing kind's bit /// position shifts. RecordingProgress, + /// Appended last for the same reason. + MaintenanceChanged, } impl EventKind { @@ -67,7 +69,7 @@ impl EventKind { /// /// The mask type below indexes into this, so the order is load-bearing: /// it is the bit order, not just a listing. - pub const ALL: [Self; 42] = [ + pub const ALL: [Self; 43] = [ Self::ServerError, Self::ServerStarted, Self::ServerShutdown, @@ -110,6 +112,7 @@ impl EventKind { Self::AuthSignInThrottled, Self::AuthPermissionDenied, Self::RecordingProgress, + Self::MaintenanceChanged, ]; /// This kind's bit position. @@ -175,6 +178,7 @@ impl EventKind { | Self::StreamProbeFailed | Self::NotificationDeadLettered | Self::ScheduledTaskFailed => Permission::SystemRead, + Self::MaintenanceChanged => Permission::MaintenanceRead, } } @@ -273,6 +277,7 @@ impl EventKind { Self::AuthSignInFailed => "auth.sign_in.failed", Self::AuthSignInThrottled => "auth.sign_in.throttled", Self::AuthPermissionDenied => "auth.permission.denied", + Self::MaintenanceChanged => "maintenance.changed", } } diff --git a/shared/src/model/event/message.rs b/shared/src/model/event/message.rs index deece278a..f584d06a1 100644 --- a/shared/src/model/event/message.rs +++ b/shared/src/model/event/message.rs @@ -4,7 +4,7 @@ use crate::model::{ notification::{registry, EventId, Severity}, stats::SourceStats, ActiveUserConnectionChange, ConfigReloadFailure, ConfigType, ConnectionDenied, DiskAlert, LibraryScanProgressEvent, - LibraryScanSummaryStatus, MetadataUpdateFailure, MsgKind, NotificationDeadLetter, Permission, + LibraryScanSummaryStatus, MaintenanceEntries, MetadataUpdateFailure, MsgKind, NotificationDeadLetter, Permission, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, PlaylistUpdateState, ProviderAccountEvent, ProviderAccountState, ProviderFetchFailure, ProviderPoolExhausted, ProviderPriorityFallback, RecordingLifecycleMessage, ScheduledTaskFailure, ServerLifecycleEvent, ServerLifecycleState, StreamProbeFailure, @@ -34,6 +34,8 @@ pub enum EventMessage { /// session can rate limit it without also delaying a state transition. RecordingProgress, RecordingRulesChanged, + /// Inputs in maintenance after a change; the whole map, it is small. + MaintenanceChanged(Arc), InputMetadataUpdatesCompleted(Arc), InputMetadataUpdatesStarted(Arc), /// A metadata update cycle ended with tasks it could not finish. @@ -126,6 +128,7 @@ impl EventMessage { Self::RecordingChanged => EventKind::RecordingChanged, Self::RecordingProgress => EventKind::RecordingProgress, Self::RecordingRulesChanged => EventKind::RecordingRulesChanged, + Self::MaintenanceChanged(_) => EventKind::MaintenanceChanged, Self::InputMetadataUpdatesCompleted(_) => EventKind::InputMetadataUpdatesCompleted, Self::InputMetadataUpdatesStarted(_) => EventKind::InputMetadataUpdatesStarted, Self::InputMetadataUpdatesFailed(_) => EventKind::InputMetadataUpdatesFailed, @@ -263,6 +266,7 @@ impl EventMessage { Self::ActiveProvider(_, _) => registry::PROVIDER_CONNECTIONS_CHANGED, Self::RecordingChanged => registry::RECORDING_QUEUE_CHANGED, Self::RecordingRulesChanged => registry::RECORDING_RULES_CHANGED, + Self::MaintenanceChanged(_) => registry::MAINTENANCE_CHANGED, Self::PlaylistUpdateProgress(_) | Self::SystemInfoUpdate(_) | Self::RecordingProgress => return None, }) } @@ -298,6 +302,7 @@ impl EventMessage { Self::SystemInfoUpdate(info) => encode(info.as_ref()), Self::LibraryScanProgress(progress) => encode(progress), Self::RecordingChanged | Self::RecordingProgress | Self::RecordingRulesChanged => serde_json::Value::Null, + Self::MaintenanceChanged(entries) => encode(entries.as_ref()), Self::InputMetadataUpdatesStarted(input) | Self::InputMetadataUpdatesCompleted(input) => { serde_json::json!({ "input": input.as_ref() }) } diff --git a/shared/src/model/event/tests.rs b/shared/src/model/event/tests.rs index db9bdfbe0..d36856c3f 100644 --- a/shared/src/model/event/tests.rs +++ b/shared/src/model/event/tests.rs @@ -62,3 +62,12 @@ fn groups_changed_alias_resolves_for_kinds() { // to the old name keep receiving group changes. assert_eq!(EventKind::from_wire_name("playlist.groups.changed"), Some(EventKind::PlaylistWatchChanged)); } + +#[test] +fn maintenance_changed_requires_maintenance_read() { + assert_eq!( + EventKind::MaintenanceChanged.required_permission(), + crate::model::permission::Permission::MaintenanceRead + ); + assert_eq!(EventKind::MaintenanceChanged.as_wire_name(), "maintenance.changed"); +} diff --git a/shared/src/model/maintenance.rs b/shared/src/model/maintenance.rs new file mode 100644 index 000000000..d83503e01 --- /dev/null +++ b/shared/src/model/maintenance.rs @@ -0,0 +1,101 @@ +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Path segment that addresses every known input at once. +pub const MAINTENANCE_WILDCARD: &str = "*"; +/// Upper bound for a maintenance remark, in characters. +pub const MAINTENANCE_REASON_MAX_CHARS: usize = 500; + +/// Why and since when an input is in maintenance. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InputMaintenance { + /// Unix timestamp in seconds; kept when the remark is updated. + pub since: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// Inputs in maintenance, keyed by canonical input name. +pub type MaintenanceEntries = BTreeMap; + +/// What the maintenance API returns after every call. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaintenanceStateDto { + pub entries: MaintenanceEntries, + /// Canonical names of all configured inputs, for the selection in the dialog. + pub inputs: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaintenanceRequestDto { + #[serde(default)] + pub reason: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MaintenanceReasonTooLong; + +impl std::fmt::Display for MaintenanceReasonTooLong { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "maintenance reason exceeds {MAINTENANCE_REASON_MAX_CHARS} characters") + } +} + +impl std::error::Error for MaintenanceReasonTooLong {} + +/// Trims the remark; an empty remark is no remark. +pub fn normalize_maintenance_reason(reason: Option<&str>) -> Result, MaintenanceReasonTooLong> { + let Some(reason) = reason.map(str::trim).filter(|reason| !reason.is_empty()) else { + return Ok(None); + }; + if reason.chars().count() > MAINTENANCE_REASON_MAX_CHARS { + return Err(MaintenanceReasonTooLong); + } + Ok(Some(reason.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reason_is_trimmed_and_empty_becomes_none() { + assert_eq!(normalize_maintenance_reason(None), Ok(None)); + assert_eq!(normalize_maintenance_reason(Some(" ")), Ok(None)); + assert_eq!(normalize_maintenance_reason(Some(" down ")), Ok(Some("down".to_string()))); + } + + #[test] + fn reason_longer_than_limit_is_rejected() { + let ok = "x".repeat(MAINTENANCE_REASON_MAX_CHARS); + let too_long = "x".repeat(MAINTENANCE_REASON_MAX_CHARS + 1); + assert_eq!(normalize_maintenance_reason(Some(&ok)), Ok(Some(ok.clone()))); + assert_eq!(normalize_maintenance_reason(Some(&too_long)), Err(MaintenanceReasonTooLong)); + } + + #[test] + fn reason_limit_counts_characters_not_bytes() { + let umlauts = "ä".repeat(MAINTENANCE_REASON_MAX_CHARS); + assert!(normalize_maintenance_reason(Some(¨auts)).is_ok()); + } + + #[test] + fn state_dto_round_trips_as_json() { + let mut entries = MaintenanceEntries::new(); + entries + .insert("provider_a".to_string(), InputMaintenance { since: 1_760_350_000, reason: Some("down".into()) }); + let dto = MaintenanceStateDto { entries, inputs: vec!["provider_a".into(), "provider_b".into()] }; + let json = serde_json::to_string(&dto).unwrap(); + assert_eq!( + json, + r#"{"entries":{"provider_a":{"since":1760350000,"reason":"down"}},"inputs":["provider_a","provider_b"]}"# + ); + assert_eq!(serde_json::from_str::(&json).unwrap(), dto); + } + + #[test] + fn missing_reason_deserializes_as_none() { + let entry: InputMaintenance = serde_json::from_str(r#"{"since":1}"#).unwrap(); + assert_eq!(entry.reason, None); + } +} diff --git a/shared/src/model/mod.rs b/shared/src/model/mod.rs index f3dd2c192..ccc549d1e 100644 --- a/shared/src/model/mod.rs +++ b/shared/src/model/mod.rs @@ -18,6 +18,7 @@ mod ip_check; mod item_field; mod library_request; mod log; +mod maintenance; mod mapping; mod media_properties; mod messaging; @@ -79,16 +80,16 @@ pub mod xtream_const; pub use self::{ active_user_connection_change::*, auth::*, auth_audit::*, cluster_flags::*, config::*, connection_denied::*, custom_video_stream_type::*, epg::*, epg_grid::*, epg_request::*, event::*, input_refresh_policy::*, - input_update_action::*, ip_check::*, item_field::*, library_request::*, log::*, mapping::*, media_properties::*, - messaging::*, metadata_update_failure::*, notification::*, notification_dead_letter::*, pagination::*, playlist::*, - playlist_categories::*, playlist_info_document::*, playlist_request::*, playlist_update_run::*, - playlist_update_status::*, processing_order::*, progress::*, provider_fetch_failure::*, provider_pool::*, - recording::*, recording_math::*, regex_cache::*, resource::*, scheduled_task_failure::*, search_fields::*, - search_request::*, server_lifecycle::*, short_epg::*, stalker::*, stalker_item::*, stats::*, status_check::*, - stream_history::*, stream_history_record::*, stream_info::*, stream_meter::*, stream_probe_failure::*, - stream_properties::*, strm_export_style::*, system_info::*, target_bouquet::*, target_type::*, transfer::*, - ui_playlist_item::*, user_command::*, user_id::*, user_lifecycle::*, uuidtype::*, watch_health::*, web_socket::*, - xtream::*, + input_update_action::*, ip_check::*, item_field::*, library_request::*, log::*, maintenance::*, mapping::*, + media_properties::*, messaging::*, metadata_update_failure::*, notification::*, notification_dead_letter::*, + pagination::*, playlist::*, playlist_categories::*, playlist_info_document::*, playlist_request::*, + playlist_update_run::*, playlist_update_status::*, processing_order::*, progress::*, provider_fetch_failure::*, + provider_pool::*, recording::*, recording_math::*, regex_cache::*, resource::*, scheduled_task_failure::*, + search_fields::*, search_request::*, server_lifecycle::*, short_epg::*, stalker::*, stalker_item::*, stats::*, + status_check::*, stream_history::*, stream_history_record::*, stream_info::*, stream_meter::*, + stream_probe_failure::*, stream_properties::*, strm_export_style::*, system_info::*, target_bouquet::*, + target_type::*, transfer::*, ui_playlist_item::*, user_command::*, user_id::*, user_lifecycle::*, uuidtype::*, + watch_health::*, web_socket::*, xtream::*, }; pub use ids::*; pub use prepare::*; diff --git a/shared/src/model/notification/registry.rs b/shared/src/model/notification/registry.rs index 3a490de7a..9ab3f4eab 100644 --- a/shared/src/model/notification/registry.rs +++ b/shared/src/model/notification/registry.rs @@ -71,6 +71,7 @@ pub const STREAM_PROBE_FAILED: EventId = EventId::new("stream.probe.failed"); // ---- dvr ------------------------------------------------------------ pub const RECORDING_QUEUE_CHANGED: EventId = EventId::new("recording.queue.changed"); pub const RECORDING_RULES_CHANGED: EventId = EventId::new("recording.rules.changed"); +pub const MAINTENANCE_CHANGED: EventId = EventId::new("maintenance.changed"); // ---- scheduler ------------------------------------------------------- pub const SCHEDULED_TASK_FAILED: EventId = EventId::new("scheduled_task.failed"); @@ -257,6 +258,11 @@ pub const ALL: &[EventDescriptor] = &[ severity: Severity::Info, description: "The recording rule set changed.", }, + EventDescriptor { + id: MAINTENANCE_CHANGED, + severity: Severity::Warn, + description: "An input was put into or taken out of maintenance mode.", + }, EventDescriptor { id: SCHEDULED_TASK_FAILED, severity: Severity::Error, diff --git a/shared/src/model/recording/metadata.rs b/shared/src/model/recording/metadata.rs index 71dc3e9fb..8aa3f4040 100644 --- a/shared/src/model/recording/metadata.rs +++ b/shared/src/model/recording/metadata.rs @@ -126,6 +126,11 @@ pub struct RecordingMetadata { /// Persisted at-most-once notification markers. #[serde(default)] pub notification_markers: Vec, + /// Set once the first provider connection was acquired. A later acquisition + /// (retry, requeue after preemption, restart) resumes the recording rather + /// than starting a new one. Internal; never projected into a DTO. + #[serde(default)] + pub provider_started: bool, /// Terminal-only previous state while a deletion is in progress. `None` /// outside the `Deleting` transitional state. pub deleting_previous_state: Option, @@ -167,6 +172,7 @@ impl RecordingMetadata { measured_bytes: 0, completed_at: None, notification_markers: Vec::new(), + provider_started: false, group: None, series_name: None, deleting_previous_state: None, diff --git a/shared/src/model/stream_history_record.rs b/shared/src/model/stream_history_record.rs index 759a130e0..96f76f356 100644 --- a/shared/src/model/stream_history_record.rs +++ b/shared/src/model/stream_history_record.rs @@ -87,6 +87,7 @@ pub enum ConnectFailureReason { Preempted, SessionExpired, Provisioning, + Maintenance, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/shared/src/model/web_socket.rs b/shared/src/model/web_socket.rs index 58eb886ba..3761c4d7c 100644 --- a/shared/src/model/web_socket.rs +++ b/shared/src/model/web_socket.rs @@ -117,6 +117,9 @@ pub enum ProtocolMessage { /// re-fetches `/api/v1/recording/rules` on receipt. No payload /// — the rule list is small and the GET is cheap. RecordingRulesChanged, + /// Inputs in maintenance after a change. Carries the whole map: it is small, + /// and a client never has to merge. + MaintenanceChangedResponse(crate::model::MaintenanceEntries), /// The socket cannot serve recordings to this session, and the reason /// is actionable. ///