Feature/failover (#924)

* feature stream failover

backend/app/src/api/model/app_state/cancel_tokens.rs

@@ -9,6 +9,8 @@ pub struct CancelTokens {
     pub(crate) qos_aggregation: CancellationToken,
     pub(crate) recordings: CancellationToken,
     pub(crate) hls_cache: CancellationToken,
+    /// Member key refresh of channel groups; lives across config reloads.
+    pub(crate) channel_groups: CancellationToken,
 }
 impl Default for CancelTokens {
     fn default() -> Self {
@@ -21,6 +23,7 @@ impl Default for CancelTokens {
             qos_aggregation: CancellationToken::new(),
             recordings: CancellationToken::new(),
             hls_cache: CancellationToken::new(),
+            channel_groups: CancellationToken::new(),
         }
     }
 }

backend/app/src/api/model/app_state/mod.rs

@@ -63,6 +63,7 @@ pub struct AppState {
     pub playlist_updates: PlaylistUpdateControl,
     /// Inputs taken out of service; read on every new stream request.
     pub maintenance: Arc<MaintenanceRegistry>,
+    pub member_health: Arc<tuliprox_session::MemberHealthIndex>,
 }
 
 impl AppState {
@@ -121,3 +122,33 @@ pub struct HdHomerunAppState {
     pub device: Arc<HdHomeRunDeviceConfig>,
     pub hd_scan_state: Arc<AtomicI8>,
 }
+
+pub struct AppQosAggregationListener {
+    pub member_health: Arc<tuliprox_session::MemberHealthIndex>,
+}
+
+impl tuliprox_session::qos_aggregation_manager::QosAggregationListener for AppQosAggregationListener {
+    fn on_snapshots(&self, snapshots: &[tuliprox_repository::QosSnapshotRecord]) {
+        let entries = snapshots.iter().map(|record| {
+            let key = if record.provider_id > 0 {
+                tuliprox_session::QosKey::Provider {
+                    input: Arc::clone(&record.input_name),
+                    provider_id: record.provider_id,
+                }
+            } else {
+                tuliprox_session::QosKey::TargetMember {
+                    target: Arc::clone(&record.target_name),
+                    member_vid: record.virtual_id,
+                }
+            };
+            let part = tuliprox_session::QosPart {
+                score_24h: record.window_24h.score,
+                score_7d: record.window_7d.score,
+                confidence: record.window_24h.confidence,
+                failure_streak: record.window_24h.successive_failure_streak,
+            };
+            (key, part)
+        });
+        self.member_health.replace_qos(entries);
+    }
+}

backend/app/src/api/model/app_state/reload.rs

@@ -17,7 +17,9 @@ use shared::{error::TuliproxError, model::WebAuthConfigDto};
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use tokio_util::sync::CancellationToken;
 use tuliprox_dvr::recording::recording_transfer::{resume_recording_worker_if_needed, spawn_recording_services};
-use tuliprox_session::{provider_dns_manager::exec_provider_dns, qos_aggregation_manager::exec_qos_aggregation};
+use tuliprox_session::{
+    provider_dns_manager::exec_provider_dns, qos_aggregation_manager::exec_qos_aggregation_with_listener,
+};
 
 macro_rules! cancel_service {
     ($field: ident, $flag:expr, $changes:expr, $cancel_tokens:expr) => {
@@ -131,6 +133,7 @@ fn cancel_services(app_state: &Arc<AppState>, changes: &UpdateChanges) {
         qos_aggregation,
         recordings,
         hls_cache: cancel_tokens.hls_cache.clone(),
+        channel_groups: cancel_tokens.channel_groups.clone(),
     };
 
     app_state.cancel_tokens.store(Arc::new(tokens));
@@ -166,7 +169,11 @@ fn start_services(app_state: &Arc<AppState>, changes: &UpdateChanges) {
         exec_provider_dns(&app_state.app_config, &app_state.cancel_tokens.load().provider_dns);
     }
     if changes.flags.contains(UpdateChangesFlags::QosAggregation) {
-        exec_qos_aggregation(&app_state.app_config, &app_state.cancel_tokens.load().qos_aggregation);
+        exec_qos_aggregation_with_listener(
+            &app_state.app_config,
+            &app_state.cancel_tokens.load().qos_aggregation,
+            Some(Arc::new(super::AppQosAggregationListener { member_health: Arc::clone(&app_state.member_health) })),
+        );
         let history_cfg =
             app_state.app_config.config.load().reverse_proxy.as_ref().and_then(|rp| rp.stream_history.clone());
         let connection_manager = Arc::clone(&app_state.connection_manager);
@@ -214,6 +221,8 @@ impl AppState {
         let storage_dir = config.storage_dir.clone();
 
         self.active_users.update_config(&config);
+        self.member_health
+            .set_policy(tuliprox_session::channel_group::CircuitPolicy::from(&config.channel_groups.circuit));
         self.app_config.set_config(config)?;
         reload_logger(config_log_level.as_deref());
         self.active_provider.update_config(&self.app_config);

backend/app/src/api/model/app_state/test_support.rs

@@ -88,6 +88,7 @@ pub fn create_test_app_state(config: Config) -> Arc<AppState> {
         auth: AuthState::for_tests(),
         playlist_updates: PlaylistUpdateControl::for_tests(),
         maintenance,
+        member_health: Arc::new(tuliprox_session::MemberHealthIndex::default()),
     })
 }
 

backend/app/src/api/model/app_state/tests.rs

@@ -276,3 +276,19 @@ async fn contactable_input_hides_inputs_in_maintenance() {
     app_state.maintenance.add("provider_a", None, 1).await.unwrap();
     assert!(app_state.contactable_input(&name).is_none(), "an input in maintenance is not contacted");
 }
+
+#[tokio::test]
+async fn config_reload_applies_channel_group_circuit_settings() {
+    let state = super::create_test_app_state(Config::default());
+    let circuit = shared::model::ChannelGroupCircuitConfig {
+        source_failures: 5,
+        ..shared::model::ChannelGroupCircuitConfig::default()
+    };
+    let config = Config {
+        channel_groups: shared::model::ChannelGroupsConfig { circuit, ..shared::model::ChannelGroupsConfig::default() },
+        ..Config::default()
+    };
+
+    assert!(state.set_config(config).await.is_ok());
+    assert_eq!(*state.member_health.policy(), tuliprox_session::channel_group::CircuitPolicy::from(&circuit));
+}