Feature/provider config (#570)

New Features
- Automatic provider failover & rotation across multiple provider URLs with a provider:// scheme and capped retries
- Configurable failover redirect patterns to influence retry/failover behavior
UI

- New interface to view and edit failover redirect patterns in reverse proxy configuration

- Expanded docs covering Provider Failover & Rotation and enriched messaging templating
Reliability

- Improved URL resolution, error reporting and logging for more robust provider handling

shared/src/model/config/input.rs

@@ -1,10 +1,10 @@
 use super::PanelApiConfigDto;
 use crate::error::{TuliproxError, TuliproxErrorKind};
 use crate::model::EpgConfigDto;
-use crate::utils::{arc_str_serde, default_as_true, deserialize_timestamp, get_credentials_from_url_str, get_trimmed_string,
-                   is_false, is_true, is_zero_u16, sanitize_sensitive_info,
-                   serialize_option_vec_flow_map_items, trim_last_slash};
-use crate::utils::{is_blank_optional_string, Internable};
+use crate::utils::{arc_str_serde, default_as_true, deserialize_timestamp, get_credentials_from_url_str,
+                   get_trimmed_string, is_false, is_true, is_zero_u16, parse_provider_scheme_url_parts,
+                   sanitize_sensitive_info, serialize_option_vec_flow_map_items, trim_last_slash, PROVIDER_SCHEME_PREFIX};
+use crate::utils::{is_blank_optional_string, Internable, arc_str_vec_serde};
 use crate::{check_input_connections, check_input_credentials, info_err_res};
 
 use enum_iterator::Sequence;
@@ -361,7 +361,7 @@ impl Default for ConfigInputDto {
 
 impl ConfigInputDto {
     #[allow(clippy::cast_possible_truncation)]
-    pub fn prepare(&mut self, index: u16, _include_computed: bool) -> Result<u16, TuliproxError> {
+    pub fn prepare(&mut self, index: u16, _include_computed: bool, provider_names: &HashSet<String>) -> Result<u16, TuliproxError> {
         self.name = self.name.trim().intern();
         if self.name.is_empty() {
             return info_err_res!("name for input is mandatory");
@@ -384,12 +384,27 @@ impl ConfigInputDto {
 
         self.persist = get_trimmed_string(self.persist.as_deref());
 
+        if self.url.starts_with(PROVIDER_SCHEME_PREFIX) {
+            if let Ok((host, _path)) = parse_provider_scheme_url_parts(&self.url) {
+                if !provider_names.contains(host) {
+                    return info_err_res!("Provider name {host} is not defined");
+                }
+            }
+        }
+
         let mut current_index = index + 1;
         self.id = current_index;
         if let Some(aliases) = self.aliases.as_mut() {
             let input_type = &self.input_type;
             for alias in aliases {
                 current_index = alias.prepare(current_index, input_type)?;
+                if alias.url.starts_with(PROVIDER_SCHEME_PREFIX) {
+                    if let Ok((host, _path)) = parse_provider_scheme_url_parts(&alias.url) {
+                        if !provider_names.contains(host) {
+                            return info_err_res!("Provider name {host} is not defined");
+                        }
+                    }
+                }
             }
         }
 
@@ -530,3 +545,26 @@ impl ConfigInputDto {
         Err(TuliproxError::new(TuliproxErrorKind::Info, format!("No matching input or alias found for input '{input_name}' with username '{username}'")))
     }
 }
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)]
+#[serde(deny_unknown_fields)]
+pub struct ConfigProviderDto {
+    #[serde(with = "arc_str_serde")]
+    pub name: Arc<str>,
+    #[serde(with = "arc_str_vec_serde")]
+    pub urls: Vec<Arc<str>>,
+}
+
+impl ConfigProviderDto {
+    pub fn prepare(&mut self) -> Result<(), TuliproxError> {
+        self.name = self.name.trim().intern();
+        if self.name.is_empty() {
+            return info_err_res!("Name for provider is mandatory");
+        }
+        self.urls = self.urls.drain(..).filter(|url| !url.trim().is_empty()).map(|u| u.trim().intern()).collect();
+        if self.urls.is_empty() {
+            return info_err_res!("Urls for provider is mandatory");
+        }
+        Ok(())
+    }
+}

shared/src/model/config/reverse_proxy.rs

@@ -1,6 +1,6 @@
 use crate::error::{TuliproxError, TuliproxErrorKind};
 use crate::model::{CacheConfigDto, GeoIpConfigDto, RateLimitConfigDto, StreamConfigDto};
-use crate::utils::{is_false, default_resource_retry_attempts,
+use crate::utils::{is_false, is_empty_optional_vec, default_resource_retry_attempts,
                    default_resource_retry_backoff_ms,
                    default_resource_retry_backoff_multiplier,
                    is_default_resource_retry_attempts,
@@ -8,6 +8,7 @@ use crate::utils::{is_false, default_resource_retry_attempts,
                    is_default_resource_retry_backoff_multiplier,
                    hex_to_u8_16};
 use log::warn;
+use crate::info_err_res;
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)]
 #[serde(deny_unknown_fields)]
@@ -117,6 +118,11 @@ impl ReverseProxyConfigDto {
                 rate_limit.prepare()?;
             }
         }
+
+        if let Some(resource_retry) = self.resource_retry.as_mut() {
+            resource_retry.prepare()?;
+        }
+
         Ok(())
     }
 }
@@ -130,6 +136,8 @@ pub struct ResourceRetryConfigDto {
     pub backoff_millis: u64,
     #[serde(default = "default_resource_retry_backoff_multiplier", skip_serializing_if = "is_default_resource_retry_backoff_multiplier")]
     pub backoff_multiplier: f64,
+    #[serde(default, skip_serializing_if = "is_empty_optional_vec")]
+    pub failover_redirect_patterns: Option<Vec<String>>,
 }
 
 impl Default for ResourceRetryConfigDto {
@@ -138,6 +146,7 @@ impl Default for ResourceRetryConfigDto {
             max_attempts: default_resource_retry_attempts(),
             backoff_millis: default_resource_retry_backoff_ms(),
             backoff_multiplier: default_resource_retry_backoff_multiplier(),
+            failover_redirect_patterns: None,
         }
     }
 }
@@ -147,5 +156,17 @@ impl ResourceRetryConfigDto {
         self.max_attempts == default_resource_retry_attempts()
             && self.backoff_millis == default_resource_retry_backoff_ms()
             && (self.backoff_multiplier - default_resource_retry_backoff_multiplier()).abs() < f64::EPSILON
+            && is_empty_optional_vec(&self.failover_redirect_patterns)
+    }
+
+    pub fn prepare(&mut self) -> Result<(), TuliproxError> {
+        if let Some(failover_redirect_patterns) = self.failover_redirect_patterns.as_mut() {
+            for pattern in failover_redirect_patterns {
+                if let Err(err) = crate::model::REGEX_CACHE.get_or_compile(pattern) {
+                    return info_err_res!("Can't parse regex: {pattern} {err}");
+                }
+            }
+        }
+        Ok(())
     }
 }

shared/src/model/config/source.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use crate::info_err_res;
 use crate::error::{TuliproxError};
 use crate::foundation::prepare_templates;
-use crate::model::{ConfigInputDto, HdHomeRunDeviceOverview, PatternTemplate};
+use crate::model::{ConfigInputDto, ConfigProviderDto, HdHomeRunDeviceOverview, PatternTemplate};
 use crate::model::config::target::ConfigTargetDto;
 use crate::utils::{arc_str_vec_serde, default_as_default, Internable};
 
@@ -35,6 +35,8 @@ impl ConfigSourceDto {
 pub struct SourcesConfigDto {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub templates: Option<Vec<PatternTemplate>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub provider: Option<Vec<ConfigProviderDto>>,
     pub inputs: Vec<ConfigInputDto>,
     pub sources: Vec<ConfigSourceDto>,
 }
@@ -42,19 +44,34 @@ pub struct SourcesConfigDto {
 impl SourcesConfigDto {
     pub fn prepare(&mut self, include_computed: bool, hdhr_config: Option<&HdHomeRunDeviceOverview>) -> Result<(), TuliproxError> {
         self.prepare_templates()?;
-        self.prepare_sources(include_computed, hdhr_config)?;
+        let provider_names =self.prepare_providers()?;
+        self.prepare_sources(include_computed, hdhr_config, &provider_names)?;
         self.check_unique_target_names()?;
         Ok(())
     }
 
-    fn prepare_sources(&mut self, include_computed: bool, hdhr_config: Option<&HdHomeRunDeviceOverview>) -> Result<(), TuliproxError> {
+    fn prepare_providers(&mut self) -> Result<HashSet<String>, TuliproxError> {
+        let mut names = HashSet::new();
+        if let Some(providers) = &mut self.provider {
+            for provider in providers {
+                provider.prepare()?;
+                if names.contains(provider.name.as_ref()) {
+                    return info_err_res!("Provider names should be unique: {}", provider.name);
+                }
+                names.insert(provider.name.to_string());
+            }
+        }
+        Ok(names)
+    }
+
+    fn prepare_sources(&mut self, include_computed: bool, hdhr_config: Option<&HdHomeRunDeviceOverview>, provider_names: &HashSet<String>) -> Result<(), TuliproxError> {
         // prepare sources and set id's
         let mut source_index: u16 = 0;
         let mut input_index: u16 = 0;
         let mut target_index: u16 = 1;
         // Prepare global inputs
         for input in &mut self.inputs {
-            input_index = input.prepare(input_index, include_computed)?;
+            input_index = input.prepare(input_index, include_computed, provider_names)?;
         }
 
         for source in &mut self.sources {