diff --git a/CHANGELOG.md b/CHANGELOG.md index 98c828482..91b35a459 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ - WebUI API-User category selection implemented. - Stream table "Copy-To-Clipboard" functions added. - Refactored provider connection handling to avoid potential race conditions. +- WebUI: Added Panel API config view and new `/api/v1/config/sources` endpoint to persist + hot-reload `sources.yml`. - Added exp_date field to inputs, aliases, and CSV batch files; accepts dates in "YYYY-MM-DD HH:MM:SS" format or Unix timestamps (seconds since epoch). - Added cloudflare_header to reverse proxy disable_header settings. - Added CPU usage to the WebUI view. @@ -35,6 +36,7 @@ - Fixed race conditions during simultaneous access by the same user. - Added extended debug logging for client requests and ID chain (request/action/virtual) to trace stream resolution. - Fixed xtream series/catchup lookups using the series-info virtual_id so episode requests now keep their own virtual_id/session. +- **NEW FEATURE `panel_api`** Added optional `panel_api` integration to renew expired accounts or provision new aliases when provider connections are exhausted. - **NEW FEATURE: Local library Module** - Comprehensive local video file scanning and metadata management - Recursive directory scanning with async tokio::fs operations - Automatic classification (Movies vs TV Series) using configurable regex patterns diff --git a/Cargo.lock b/Cargo.lock index b65404852..7abe5ea12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1096,7 +1096,7 @@ dependencies = [ [[package]] name = "frontend" -version = "3.2.21" +version = "3.2.22" dependencies = [ "anyhow", "base64", @@ -3793,7 +3793,7 @@ dependencies = [ [[package]] name = "shared" -version = "3.2.21" +version = "3.2.22" dependencies = [ "base64", "bitflags 2.10.0", @@ -4356,7 +4356,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tuliprox" -version = "3.2.21" +version = "3.2.22" dependencies = [ "arc-swap", "async-compression", diff --git a/README.md b/README.md index 037bfec44..2cece4210 100644 --- a/README.md +++ b/README.md @@ -788,7 +788,8 @@ Each input has the following attributes: - `method` can be `GET` or `POST` - `username` only mandatory for type `xtream` - `password` only mandatory for type `xtream` -- `exp_date` optional, i a date as "YYYY-MM-DD HH:MM:SS" format like `2028-11-30 12:34:12` or Unix timestamp (seconds since epoch) +- `panel_api` _optional_ for provider panel api operations +- `exp_date` optional, is a date as "YYYY-MM-DD HH:MM:SS" format like `2028-11-30 12:34:12` or Unix timestamp (seconds since epoch) - `options` is optional, + `xtream_skip_live` true or false, live section can be skipped. + `xtream_skip_vod` true or false, vod section can be skipped. @@ -970,6 +971,86 @@ The `exp_date` field is a date as: - or Unix timestamp (seconds since epoch) +#### `panel_api` + +If provider connections are exhausted, tuliprox can optionally call a provider panel API to: +- renew expired accounts first (based on `exp_date`) +- otherwise create a new alias account and persist it + +**Important!** Panel api accounts are not considering unlimited provider access! + +The API is configured generically via predefined query parameters; only `type: m3u` is supported. + +Use the literal value `auto` to fill sensitive values at runtime: +- `enabled`, optional, default true +- `api_key: auto` is replaced by `panel_api.api_key` +- in `client_renew`, `username: auto` / `password: auto` are replaced by the account being renewed +- in `client_info`, `username: auto` / `password: auto` are replaced by the account being queried + +`client_info` is used to fetch the exact `exp_date` (via the `expire` field) and is also executed on boot to sync `exp_date` for existing inputs/aliases. + +For `client_new`, the Panel API call would look like this in the example shown: + +```text +https://panel.example.tld/api.php?action=new&type=m3u&sub=1&api_key=1234567890 +``` + +Response evaluation logic +Tuliprox evaluates Panel API responses as JSON with the following logic, depending on the operation: + +`Common rule (all operations)` +- The response must contain `status: true`. If status is missing or not true, the operation is treated as failed. + +`client_new (create alias)` + +- Require `status: true`. +- Attempt to extract credentials directly from the JSON response: + - username + - password +- If one or both fields are missing, tuliprox attempts a fallback extraction from a URL contained in the JSON: + - If the JSON contains a url field, tuliprox parses it and tries to extract username/password from it (e.g., query string or embedded credentials depending on the provider’s URL format). +- If credentials cannot be derived from either the direct fields or the url fallback, the operation is treated as failed and no alias is persisted. + +`client_renew (renew existing account)` + +- Only `status: true` is evaluated. No credentials are extracted or updated as part of renew. + +`client_info (sync expiration)` + +- Require `status: true`. +- Extract the expiration timestamp/date from the JSON field: + - `expire` → used to populate/update exp_date for the corresponding input/alias. +```yaml +- sources: +- inputs: + - type: xtream + name: my_provider + url: 'http://provider.net' + username: xyz + password: secret1 + panel_api: + url: 'https://panel.example.tld/api.php' + api_key: '1234567890' + query_parameter: + client_info: + - { key: action, value: client_info } + - { key: username, value: auto } + - { key: password, value: auto } + - { key: api_key, value: auto } + client_new: + - { key: action, value: new } + - { key: type, value: m3u } + - { key: sub, value: '1' } + - { key: api_key, value: auto } + client_renew: + - { key: action, value: renew } + - { key: type, value: m3u } + - { key: username, value: auto } + - { key: password, value: auto } + - { key: sub, value: '1' } + - { key: api_key, value: auto } +``` + ### 2.2.2 `targets` Has the following top level entries: - `enabled` _optional_ default is `true`, if you disable the processing is skipped @@ -1028,7 +1109,6 @@ Channels within the `Freetv` group are first sorted by `quality` (as matched by To sort by specific parts of the content, use named capture groups such as `c1`, `c2`, `c3`, etc. The numeric suffix indicates the priority: `c1` is evaluated first, followed by `c2`, and so on. - ### 2.2.2.2 `output` Is a list of output format: diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9c99059e8..754e56369 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "tuliprox" -version = "3.2.21" +version = "3.2.22" edition = "2021" +rust-version = "1.87.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index 2cee4b7f0..f14fd7f50 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -9,6 +9,7 @@ use crate::api::model::{ }; use crate::api::model::{tee_stream, UserSession}; use crate::api::model::{ProviderAllocation, ProviderConfig, ProviderStreamState, StreamDetails, StreamingStrategy}; +use crate::api::panel_api::try_provision_account_on_exhausted; use crate::model::{ConfigInput, ResourceRetryConfig}; use crate::model::{ConfigTarget, ProxyUserCredentials}; use crate::tools::lru_cache::LRUResourceCache; @@ -31,9 +32,9 @@ use shared::utils::{ extract_extension_from_url, replace_url_extension, sanitize_sensitive_info, DASH_EXT, HLS_EXT, }; use std::borrow::Cow; -use std::collections::HashMap; use std::io::SeekFrom; use std::path::{Path, PathBuf}; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncSeekExt}; @@ -177,7 +178,7 @@ pub async fn serve_file(file_path: &Path, mime_type: mime::Mime) -> impl IntoRes pub fn get_user_target_by_username( username: &str, - app_state: &AppState, + app_state: &Arc, ) -> Option<(ProxyUserCredentials, Arc)> { if !username.is_empty() { return app_state.app_config.get_target_for_username(username); @@ -236,7 +237,7 @@ pub struct StreamOptions { /// from the provider without additional handling. /// /// Returns a `StreamOptions` instance with the resolved configuration. -fn get_stream_options(app_state: &AppState) -> StreamOptions { +fn get_stream_options(app_state: &Arc) -> StreamOptions { let (stream_retry, buffer_enabled, buffer_size) = app_state .app_config .config @@ -291,7 +292,7 @@ pub fn get_stream_alternative_url( } async fn get_redirect_alternative_url<'a>( - app_state: &AppState, + app_state: &Arc, redirect_url: &'a str, input: &ConfigInput, ) -> Cow<'a, str> { @@ -338,18 +339,22 @@ async fn get_redirect_alternative_url<'a>( /// /// This logic helps abstract the decision-making behind provider selection and stream URL resolution. async fn resolve_streaming_strategy( - app_state: &AppState, + app_state: &Arc, stream_url: &str, fingerprint: &Fingerprint, input: &ConfigInput, force_provider: Option<&str>, ) -> StreamingStrategy { // allocate a provider connection - let provider_connection_handle = match force_provider { + let mut provider_connection_handle = match force_provider { Some(provider) => app_state.active_provider.force_exact_acquire_connection(provider, &fingerprint.addr).await, None => app_state.active_provider.acquire_connection(&input.name, &fingerprint.addr).await, }; + if provider_connection_handle.is_none() && force_provider.is_none() && input.panel_api.as_ref().is_some_and(|panel_api| panel_api.enabled) { + provider_connection_handle = try_provision_and_reacquire_on_provider_pool_exhausted(app_state, fingerprint, input).await; + } + let stream_response_params = if let Some(allocation) = provider_connection_handle.as_ref().map(|ph| &ph.allocation) { match allocation { @@ -358,26 +363,32 @@ async fn resolve_streaming_strategy( let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); ProviderStreamState::Custom(stream) } - ProviderAllocation::Available(ref provider) - | ProviderAllocation::GracePeriod(ref provider) => { + ProviderAllocation::Available(ref provider_cfg) + | ProviderAllocation::GracePeriod(ref provider_cfg) => { // force_stream_provider means we keep the url and the provider. // If force_stream_provider or the input is the same as the config we don't need to get new url - let (provider, url) = if force_provider.is_some() || provider.id == input.id { + let (selected_provider_name, url) = if force_provider.is_some() || provider_cfg.id == input.id { (input.name.clone(), stream_url.to_string()) } else { - ( - provider.name.clone(), - get_stream_alternative_url(stream_url, input, provider), - ) + (provider_cfg.name.clone(), get_stream_alternative_url(stream_url, input, provider_cfg)) }; + debug_if_enabled!( + "provider session: input={} provider_cfg={} user={} allocation={} stream_url={}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&provider_cfg.name), + sanitize_sensitive_info(provider_cfg.get_user_info().as_ref().map_or_else(|| "?", |u| u.username.as_str())), + allocation.short_key(), + sanitize_sensitive_info(&url) + ); + match allocation { ProviderAllocation::Exhausted => { let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); ProviderStreamState::Custom(stream) - } - ProviderAllocation::Available(_) => ProviderStreamState::Available(Some(provider), url), - ProviderAllocation::GracePeriod(_) => ProviderStreamState::GracePeriod(Some(provider), url), + }, + ProviderAllocation::Available(_) => ProviderStreamState::Available(Some(selected_provider_name), url), + ProviderAllocation::GracePeriod(_) => ProviderStreamState::GracePeriod(Some(selected_provider_name), url), } } } @@ -395,6 +406,40 @@ async fn resolve_streaming_strategy( } } +async fn try_provision_and_reacquire_on_provider_pool_exhausted( + app_state: &Arc, + fingerprint: &Fingerprint, + input: &ConfigInput, +) -> Option { + let active_provider_connections = app_state + .active_provider + .active_connections() + .await + .map(|c| c.into_iter().collect::>()); + debug_if_enabled!( + "panel_api: provider pool exhausted for input {} (active_provider_connections={:?})", + sanitize_sensitive_info(&input.name), + active_provider_connections + ); + + if try_provision_account_on_exhausted(app_state, input).await { + debug_if_enabled!( + "panel_api: provider pool exhausted for input {}, provision succeeded; re-acquiring connection", + sanitize_sensitive_info(&input.name) + ); + app_state + .active_provider + .acquire_connection(&input.name, &fingerprint.addr) + .await + } else { + debug_if_enabled!( + "panel_api: provider pool exhausted for input {}, provision skipped/failed", + sanitize_sensitive_info(&input.name) + ); + None + } +} + fn get_grace_period_millis( connection_permission: UserConnectionPermission, stream_response_params: &ProviderStreamState, @@ -438,27 +483,12 @@ async fn create_stream_response_details( &streaming_strategy.provider_stream_state, config_grace_period_millis, ); + let guard_provider_name = streaming_strategy .provider_handle .as_ref() .and_then(|guard| guard.allocation.get_provider_name()); - if let ProviderStreamState::GracePeriod(ref provider_grace_check, ref stream_url) = streaming_strategy.provider_stream_state { - tokio::time::sleep(tokio::time::Duration::from_millis(grace_period_millis)).await; - if let Some(provider_name) = provider_grace_check.as_ref() { - if app_state.active_provider.is_over_limit(provider_name).await { - app_state.connection_manager.update_stream_detail(&fingerprint.addr, CustomVideoStreamType::ProviderConnectionsExhausted).await; - debug!("Provider connections exhausted after grace period for provider: {provider_name}"); - let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - streaming_strategy.provider_stream_state = ProviderStreamState::Custom(stream); - let provider_handle = streaming_strategy.provider_handle.take(); - app_state.connection_manager.release_provider_handle(provider_handle).await; - } - } else { - streaming_strategy.provider_stream_state = ProviderStreamState::Available(provider_grace_check.clone(), stream_url.clone()); - } - } - match streaming_strategy.provider_stream_state { // custom stream means we display our own stream like connection exhausted, channel-unavailable... ProviderStreamState::Custom(provider_stream) => { @@ -577,7 +607,7 @@ where } pub async fn redirect_response<'a, P>( - app_state: &AppState, + app_state: &Arc, params: &'a RedirectParams<'a, P>, ) -> Option where @@ -700,7 +730,7 @@ fn is_throttled_stream(item_type: PlaylistItemType, throttle_kbps: usize) -> boo } fn prepare_body_stream( - app_state: &AppState, + app_state: &Arc, item_type: PlaylistItemType, stream: S, ) -> axum::body::Body @@ -962,7 +992,7 @@ pub async fn stream_response( axum::http::StatusCode::BAD_REQUEST.into_response() } -fn get_stream_throttle(app_state: &AppState) -> u64 { +fn get_stream_throttle(app_state: &Arc) -> u64 { app_state .app_config .config @@ -1197,7 +1227,7 @@ fn get_add_cache_content( } async fn build_stream_response( - app_state: &AppState, + app_state: &Arc, resource_url: &str, response: reqwest::Response, ) -> axum::response::Response { @@ -1257,7 +1287,7 @@ async fn build_stream_response( } async fn fetch_resource_with_retry( - app_state: &AppState, + app_state: &Arc, url: &Url, resource_url: &str, req_headers: &HashMap>, @@ -1361,7 +1391,7 @@ fn calculate_retry_backoff(base_delay_ms: u64, multiplier: f64, attempt: u32) -> /// # Panics pub async fn resource_response( - app_state: &AppState, + app_state: &Arc, resource_url: &str, req_headers: &HeaderMap, input: Option<&ConfigInput>, diff --git a/backend/src/api/config_watch.rs b/backend/src/api/config_watch.rs index 31ba94b75..183ddfc93 100644 --- a/backend/src/api/config_watch.rs +++ b/backend/src/api/config_watch.rs @@ -14,7 +14,7 @@ use arc_swap::access::Access; use arc_swap::ArcSwap; use tokio_util::sync::CancellationToken; -enum ConfigFile { +pub enum ConfigFile { Config, ApiProxy, Mapping, @@ -77,7 +77,7 @@ impl ConfigFile { Ok(()) } - async fn load_sources(app_state: &Arc) -> Result<(), TuliproxError> { + pub async fn load_sources(app_state: &Arc) -> Result<(), TuliproxError> { let paths = > as Access>::load(&app_state.app_config.paths); let sources_file = paths.sources_file_path.as_str(); let config = > as Access>::load(&app_state.app_config.config); diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index 7834e430a..0e17476b0 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -3,7 +3,7 @@ use crate::model::{ApiProxyConfig, InputSource}; use axum::response::IntoResponse; use axum::Router; use serde_json::json; -use shared::model::{ApiProxyConfigDto, ApiProxyServerInfoDto, ConfigDto}; +use shared::model::{ApiProxyConfigDto, ApiProxyServerInfoDto, ConfigDto, SourcesConfigDto}; use std::sync::Arc; use log::error; use shared::error::TuliproxError; @@ -52,6 +52,38 @@ async fn save_config_main( } } +async fn save_config_sources( + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(mut sources): axum::extract::Json, +) -> impl axum::response::IntoResponse + Send { + if let Err(err) = sources.prepare(false, None) { + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response(); + } + let paths = app_state.app_config.paths.load(); + let file_path = paths.sources_file_path.as_str(); + let config = app_state.app_config.config.load(); + let backup_dir = config.get_backup_dir(); + match utils::save_sources_config(file_path, backup_dir.as_ref(), &sources).await { + Ok(()) => {} + Err(err) => { + error!("Failed to save source.yml {err}"); + return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.to_string()}))).into_response(); + } + } + + // update runtime (hot reload semantics like panel_api reload) + match crate::model::SourcesConfig::try_from(&sources) { + Ok(src) => { + if let Err(err) = app_state.app_config.set_sources(src) { + return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.to_string()}))).into_response(); + } + app_state.active_provider.update_config(&app_state.app_config); + axum::http::StatusCode::OK.into_response() + } + Err(err) => (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response(), + } +} + async fn save_config_api_proxy_config( axum::extract::State(app_state): axum::extract::State>, axum::extract::Json(mut req_api_proxy): axum::extract::Json>, @@ -143,5 +175,6 @@ pub fn v1_api_config_register(router: Router>) -> axum::Router Self { - Self { client_id, allocation } + pub fn new(client_id: ClientConnectionId, allocation_id: AllocationId, allocation: ProviderAllocation) -> Self { + Self { + client_id, + allocation_id, + allocation, + } } } #[derive(Debug, Clone)] struct SharedAllocation { + allocation_id: AllocationId, allocation: ProviderAllocation, connections: HashSet, } @@ -38,13 +45,14 @@ struct SharedConnections { #[derive(Debug, Clone, Default)] struct Connections { - single: HashMap, + single: HashMap>, shared: SharedConnections, } pub struct ActiveProviderManager { providers: ProviderLineupManager, connections: RwLock, + next_allocation_id: AtomicU64, } impl ActiveProviderManager { @@ -54,6 +62,7 @@ impl ActiveProviderManager { Self { providers: ProviderLineupManager::new(inputs, grace_period_millis, grace_period_timeout_secs, event_manager), connections: RwLock::new(Connections::default()), + next_allocation_id: AtomicU64::new(1), } } @@ -69,37 +78,49 @@ impl ActiveProviderManager { (grace_period_millis, grace_period_timeout_secs) } - pub async fn update_config(&self, cfg: &AppConfig) { + pub fn update_config(&self, cfg: &AppConfig) { let (grace_period_millis, grace_period_timeout_secs) = Self::get_grace_options(cfg); let inputs = Self::get_config_inputs(cfg); - self.providers.update_config(inputs, grace_period_millis, grace_period_timeout_secs).await; + self.providers.update_config(inputs, grace_period_millis, grace_period_timeout_secs); } - async fn acquire_connection_inner(&self, provider_or_input_name: &str, addr: &SocketAddr, force: bool) -> Option { + async fn acquire_connection_inner( + &self, + provider_or_input_name: &str, + addr: &SocketAddr, + force: bool, + allow_grace_override: Option, + ) -> Option { // Call the specific acquisition function let allocation = if force { self.providers.force_exact_acquire_connection(provider_or_input_name).await } else { - self.providers.acquire_connection(provider_or_input_name).await + match allow_grace_override { + Some(allow_grace) => { + self.providers + .acquire_connection_with_grace_override(provider_or_input_name, allow_grace) + .await + } + None => self.providers.acquire_connection(provider_or_input_name).await, + } }; match &allocation { ProviderAllocation::Exhausted => {} ProviderAllocation::Available(_) | ProviderAllocation::GracePeriod(_) => { let provider_name = allocation.get_provider_name().unwrap_or_default(); + let allocation_id = self.next_allocation_id.fetch_add(1, Ordering::Relaxed); let mut connections = self.connections.write().await; - if let Some(old) = connections.single.insert(*addr, allocation.clone()) { + let per_addr = connections.single.entry(*addr).or_default(); + if !per_addr.is_empty() { trace_if_enabled!( - "register_connection: address {} already had a allocation for provider {:?} — forcing release on the old allocation", - sanitize_sensitive_info(&addr.to_string()), - old.get_provider_name().unwrap_or_default()); - - drop(connections); - old.release().await; + "register_connection: address {addr} already has {} provider allocations", + per_addr.len() + ); } - - debug_if_enabled!("Added provider connection {provider_name:?} for {}", sanitize_sensitive_info(&addr.to_string())); - return Some(ProviderHandle::new(*addr, allocation)); + per_addr.insert(allocation_id, allocation.clone()); + debug_if_enabled!("Added provider connection {provider_name:?} for {}", sanitize_sensitive_info(&addr.to_string())); + return Some(ProviderHandle::new(*addr, allocation_id, allocation)); } } @@ -107,12 +128,22 @@ impl ActiveProviderManager { } pub async fn force_exact_acquire_connection(&self, provider_name: &str, addr: &SocketAddr) -> Option { - self.acquire_connection_inner(provider_name, addr, true).await + self.acquire_connection_inner(provider_name, addr, true, None).await } // Returns the next available provider connection pub async fn acquire_connection(&self, input_name: &str, addr: &SocketAddr) -> Option { - self.acquire_connection_inner(input_name, addr, false).await + self.acquire_connection_inner(input_name, addr, false, None).await + } + + /// Acquire a provider connection while optionally disabling provider grace allocations. + pub async fn acquire_connection_with_grace_override( + &self, + input_name: &str, + addr: &SocketAddr, + allow_grace: bool, + ) -> Option { + self.acquire_connection_inner(input_name, addr, false, Some(allow_grace)).await } // This method is used for redirects to cycle through provider @@ -130,18 +161,20 @@ impl ActiveProviderManager { pub async fn release_connection(&self, addr: &SocketAddr) { // Single connection - let single_allocation = { + let single_allocations = { let mut connections = self.connections.write().await; connections.single.remove(addr) }; - if let Some(allocation) = single_allocation { - debug_if_enabled!( - "Released provider connection {:?} for {}", - allocation.get_provider_name().unwrap_or_default(), - sanitize_sensitive_info(&addr.to_string()) - ); - allocation.release().await; + if let Some(allocations) = single_allocations { + for (_id, allocation) in allocations { + debug_if_enabled!( + "Released provider connection {:?} for {}", + allocation.get_provider_name().unwrap_or_default(), + sanitize_sensitive_info(&addr.to_string()) + ); + allocation.release().await; + } return; } @@ -188,31 +221,109 @@ impl ActiveProviderManager { } pub async fn release_handle(&self, handle: &ProviderHandle) { - self.release_connection(&handle.client_id).await; + let mut released = None; + { + let mut connections = self.connections.write().await; + if let Some(per_addr) = connections.single.get_mut(&handle.client_id) { + released = per_addr.remove(&handle.allocation_id); + if per_addr.is_empty() { + connections.single.remove(&handle.client_id); + } + } + + if released.is_none() { + let mut remove_key: Option = None; + // TODO O(n) over all keys, maybe better approach ist to use a Hashmap shared_by_allocation_id: HashMap + for (key, shared) in &connections.shared.by_key { + if shared.allocation_id == handle.allocation_id { + remove_key = Some(key.clone()); + break; + } + } + + if let Some(key) = remove_key { + if let Some(shared) = connections.shared.by_key.remove(&key) { + released = Some(shared.allocation); + for addr in shared.connections { + connections.shared.key_by_addr.remove(&addr); + } + } + } + } + } + + if let Some(allocation) = released { + allocation.release().await; + } } pub async fn make_shared_connection(&self, addr: &SocketAddr, key: &str) { - let mut connections = self.connections.write().await; - let handle = connections.single.remove(addr); - if let Some(allocation) = handle { - debug_if_enabled!("Shared connection: Promoted connection {} to shared with key {key:?}", sanitize_sensitive_info(&addr.to_string())); - connections.shared.by_key.insert(key.to_string(), SharedAllocation { allocation, connections: HashSet::from([*addr]) }); - connections.shared.key_by_addr.insert(*addr, key.to_string()); + let extras = { + let mut connections = self.connections.write().await; + let mut extras = Vec::new(); + let handle = connections.single.remove(addr).and_then(|m| { + if m.is_empty() { + return None; + } + let mut iter = m.into_iter(); + let (id, allocation) = iter.next().expect("non-empty map"); + for (_extra_id, extra_alloc) in iter { + extras.push(extra_alloc); + } + if !extras.is_empty() { + trace_if_enabled!( + "Shared connection promotion expects a single allocation for {addr}, found {}", + extras.len() + 1 + ); + } + Some(ProviderHandle::new(*addr, id, allocation)) + }); + + if let Some(handle) = &handle { + let provider_name = handle.allocation.get_provider_name().unwrap_or_default(); + debug_if_enabled!( + "Shared connection: promoted addr {addr} provider={} key={}", + sanitize_sensitive_info(&provider_name), + sanitize_sensitive_info(key) + ); + connections.shared.by_key.insert( + key.to_string(), + SharedAllocation { + allocation_id: handle.allocation_id, + allocation: handle.allocation.clone(), + connections: HashSet::from([*addr]), + }, + ); + connections.shared.key_by_addr.insert(*addr, key.to_string()); + } + extras + }; + + for allocation in extras { + allocation.release().await; } } pub async fn add_shared_connection(&self, addr: &SocketAddr, key: &str) { let mut connections = self.connections.write().await; if let Some(shared_allocation) = connections.shared.by_key.get_mut(key) { - debug_if_enabled!("Shared connection: Added connection {} to shared with key {key:?}", sanitize_sensitive_info(&addr.to_string())); + let provider_name = shared_allocation.allocation.get_provider_name().unwrap_or_default(); + debug_if_enabled!( + "Shared connection: added addr {addr} provider={} key={}", + sanitize_sensitive_info(&provider_name), + sanitize_sensitive_info(key) + ); shared_allocation.connections.insert(*addr); connections.shared.key_by_addr.insert(*addr, key.to_string()); } else { - error!("Failed to add shared connection for {}: url: {key:?} not found", sanitize_sensitive_info(&addr.to_string())); + error!( + "Failed to add shared connection for {addr}: url {} not found", + sanitize_sensitive_info(key) + ); } } pub async fn get_provider_connections_count(&self) -> usize { self.providers.active_connection_count().await } -} \ No newline at end of file +} diff --git a/backend/src/api/model/app_state.rs b/backend/src/api/model/app_state.rs index 4994656ab..07ed25904 100644 --- a/backend/src/api/model/app_state.rs +++ b/backend/src/api/model/app_state.rs @@ -120,7 +120,7 @@ pub async fn update_app_state_sources( ) -> Result<(), TuliproxError> { let targets = sources.validate_targets(Some(&app_state.forced_targets.load().target_names))?; app_state.forced_targets.store(Arc::new(targets)); - let updates = app_state.set_sources(sources).await?; + let updates = app_state.set_sources(sources)?; update_target_caches(app_state, updates.targets.as_ref()).await; restart_services(app_state, &updates); Ok(()) @@ -278,9 +278,7 @@ impl AppState { self.active_users.update_config(&config); self.app_config.set_config(config)?; - self.active_provider - .update_config(&self.app_config) - .await; + self.active_provider.update_config(&self.app_config); self.update_config().await; if changes.geoip { @@ -323,12 +321,10 @@ impl AppState { } } - pub(in crate::api::model) async fn set_sources(&self,sources: SourcesConfig) -> Result { + pub(in crate::api::model) fn set_sources(&self,sources: SourcesConfig) -> Result { let changes = self.detect_changes_for_sources(&sources); self.app_config.set_sources(sources)?; - self.active_provider - .update_config(&self.app_config) - .await; + self.active_provider.update_config(&self.app_config); Ok(changes) } diff --git a/backend/src/api/model/connection_manager.rs b/backend/src/api/model/connection_manager.rs index 54aeca8f8..a6303435b 100644 --- a/backend/src/api/model/connection_manager.rs +++ b/backend/src/api/model/connection_manager.rs @@ -65,7 +65,7 @@ impl ConnectionManager { pub async fn release_provider_handle(&self, provider_handle: Option) { if let Some(handle) = provider_handle { - self.release_provider_connection(&handle.client_id).await; + self.provider_manager.release_handle(&handle).await; } } diff --git a/backend/src/api/model/provider_config.rs b/backend/src/api/model/provider_config.rs index e69376aa4..667b21bfe 100644 --- a/backend/src/api/model/provider_config.rs +++ b/backend/src/api/model/provider_config.rs @@ -6,8 +6,10 @@ use std::ops::Deref; use std::sync::{Arc}; use tokio::sync::RwLock; use shared::model::InputType; +use shared::utils::sanitize_sensitive_info; use shared::write_if_some; use crate::api::model::ProviderAllocation; +use crate::utils::debug_if_enabled; pub type ProviderConnectionChangeCallback = Arc; @@ -43,7 +45,7 @@ pub struct ProviderConfig { max_connections: usize, priority: i16, exp_date: Option, - connection: RwLock, + connection: Arc>, on_connection_change: ProviderConnectionChangeCallback, } @@ -99,10 +101,18 @@ macro_rules! modify_connections { } impl ProviderConfig { - pub fn new<'a, F>(cfg: &ConfigInput, get_connection: Option, on_connection_change: ProviderConnectionChangeCallback) -> Self - where - F: Fn(&str) -> Option<&'a ProviderConfigConnection>, - { + pub fn new(cfg: &ConfigInput, connection: Arc>, on_connection_change: ProviderConnectionChangeCallback) -> Self { + let panel_api_enabled = cfg.panel_api.as_ref().is_some_and(|panel_api| panel_api.enabled); + // Logic change: panel api accounts are not considering unlimited provider access! + let effective_max_connections = if panel_api_enabled && cfg.max_connections == 0 { + debug_if_enabled!( + "panel_api: input '{}' has max_connections=0; defaulting effective max_connections to 1 for pool accounting", + cfg.name + ); + 1usize + } else { + cfg.max_connections as usize + }; Self { id: cfg.id, name: cfg.name.clone(), @@ -110,18 +120,30 @@ impl ProviderConfig { username: cfg.username.clone(), password: cfg.password.clone(), input_type: cfg.input_type, - max_connections: cfg.max_connections as usize, + max_connections: effective_max_connections, priority: cfg.priority, exp_date: cfg.exp_date, - connection: RwLock::new(get_connection.and_then(|f| f(cfg.name.as_str())).map_or_else(Default::default, Clone::clone)), + connection, on_connection_change } } - pub fn new_alias<'a, F>(cfg: &ConfigInput, alias: &ConfigInputAlias, get_connection: Option, on_connection_change: ProviderConnectionChangeCallback) -> Self - where - F: Fn(&str) -> Option<&'a ProviderConfigConnection>, - { + pub fn new_alias( + cfg: &ConfigInput, + alias: &ConfigInputAlias, + connection: Arc>, + on_connection_change: ProviderConnectionChangeCallback, + ) -> Self { + let panel_api_enabled = cfg.panel_api.as_ref().is_some_and(|panel_api| panel_api.enabled); + let effective_max_connections = if panel_api_enabled && alias.max_connections == 0 { + debug_if_enabled!( + "panel_api: alias '{}' has max_connections=0; defaulting effective max_connections to 1 for pool accounting", + alias.name + ); + 1usize + } else { + alias.max_connections as usize + }; Self { id: alias.id, name: alias.name.clone(), @@ -129,14 +151,24 @@ impl ProviderConfig { username: alias.username.clone(), password: alias.password.clone(), input_type: cfg.input_type, - max_connections: alias.max_connections as usize, + max_connections: effective_max_connections, priority: alias.priority, exp_date: alias.exp_date, - connection: RwLock::new(get_connection.and_then(|f| f(alias.name.as_str())).map_or_else(Default::default, Clone::clone)), + connection, on_connection_change, } } + #[inline] + pub fn max_connections(&self) -> usize { + self.max_connections + } + + #[inline] + pub(crate) fn exp_date(&self) -> Option { + self.exp_date + } + pub fn get_user_info(&self) -> Option { InputUserInfo::new(self.input_type, self.username.as_deref(), self.password.as_deref(), &self.url) } @@ -222,6 +254,12 @@ impl ProviderConfig { guard.granted_grace = false; guard.grace_ts = 0; } + debug_if_enabled!( + "Provider {} granting grace allocation (current_connections={}, max_connections={})", + sanitize_sensitive_info(&self.name), + connections, + self.max_connections + ); guard.granted_grace = true; guard.grace_ts = now; modify_connections!(self, guard, +1); @@ -326,15 +364,6 @@ impl ProviderConfigWrapper { } None } - - pub async fn get_connection_info(&self) -> ProviderConfigConnection { - let guard = self.inner.connection.read().await; - ProviderConfigConnection { - current_connections: guard.current_connections, - granted_grace: guard.granted_grace, - grace_ts: guard.grace_ts, - } - } } impl Deref for ProviderConfigWrapper { type Target = ProviderConfig; @@ -342,4 +371,4 @@ impl Deref for ProviderConfigWrapper { fn deref(&self) -> &Self::Target { &self.inner } -} \ No newline at end of file +} diff --git a/backend/src/api/model/provider_lineup_manager.rs b/backend/src/api/model/provider_lineup_manager.rs index b25455cce..4514a9e69 100644 --- a/backend/src/api/model/provider_lineup_manager.rs +++ b/backend/src/api/model/provider_lineup_manager.rs @@ -1,14 +1,16 @@ use crate::api::model::provider_config::ProviderConfigWrapper; use crate::api::model::{EventManager, ProviderConfig, ProviderConfigConnection, ProviderConnectionChangeCallback}; +use crate::model::{is_input_expired, ConfigInput}; use crate::utils::debug_if_enabled; -use crate::model::ConfigInput; use arc_swap::ArcSwap; +use dashmap::DashMap; use log::{debug, log_enabled}; use shared::utils::{display_vec, sanitize_sensitive_info}; use std::collections::HashMap; use std::fmt; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; +use tokio::sync::RwLock; macro_rules! gen_provider_search { ($fn_name:ident, $field: ident, $crit_type:ty) => { @@ -45,6 +47,16 @@ macro_rules! gen_provider_search { } } +fn get_or_create_provider_connection( + provider_connections: &DashMap>>, + provider_name: &str, +) -> Arc> { + provider_connections + .entry(provider_name.to_string()) + .or_insert_with(|| Arc::new(RwLock::new(ProviderConfigConnection::default()))) + .clone() +} + #[derive(Debug, Clone)] pub enum ProviderAllocation { Exhausted, @@ -53,6 +65,14 @@ pub enum ProviderAllocation { } impl ProviderAllocation { + pub fn short_key(&self) -> &str { + match self { + ProviderAllocation::Exhausted => "exhausted", + ProviderAllocation::Available(_) => "available", + ProviderAllocation::GracePeriod(_) => "grace_period", + } + } + pub fn new_available(config: Arc) -> Self { ProviderAllocation::Available(config) } @@ -165,12 +185,9 @@ struct SingleProviderLineup { } impl SingleProviderLineup { - fn new<'a, F>(cfg: &ConfigInput, get_connection: Option, connection_change: &ProviderConnectionChangeCallback) -> Self - where - F: Fn(&str) -> Option<&'a ProviderConfigConnection>, - { + fn new(cfg: &ConfigInput, connection: Arc>, connection_change: &ProviderConnectionChangeCallback) -> Self { Self { - provider: ProviderConfigWrapper::new(ProviderConfig::new(cfg, get_connection, Arc::clone(connection_change))), + provider: ProviderConfigWrapper::new(ProviderConfig::new(cfg, connection, Arc::clone(connection_change))), } } @@ -214,41 +231,31 @@ impl fmt::Display for ProviderPriorityGroup { } } -impl ProviderPriorityGroup { - async fn is_exhausted(&self) -> bool { - match self { - ProviderPriorityGroup::SingleProviderGroup(g) => g.is_exhausted().await, - ProviderPriorityGroup::MultiProviderGroup(_, groups) => { - for g in groups { - if !g.is_exhausted().await { - return false; - } - } - true - } - } - } -} - - /// Manages multiple providers, ensuring that connections are allocated in a round-robin manner based on priority. #[repr(align(64))] #[derive(Debug)] struct MultiProviderLineup { name: String, providers: Vec, - index: AtomicUsize, } impl MultiProviderLineup { - pub fn new<'a, F>(cfg_input: &ConfigInput, get_connection: Option, connection_change: &ProviderConnectionChangeCallback) -> Self - where - F: Fn(&str) -> Option<&'a ProviderConfigConnection> + Copy, - { - let mut inputs = vec![ProviderConfigWrapper::new(ProviderConfig::new(cfg_input, get_connection, Arc::clone(connection_change)))]; + pub fn new( + cfg_input: &ConfigInput, + provider_connections: &DashMap>>, + connection_change: &ProviderConnectionChangeCallback, + ) -> Self { + let input_connection = get_or_create_provider_connection(provider_connections, cfg_input.name.as_str()); + let mut inputs = vec![ProviderConfigWrapper::new(ProviderConfig::new(cfg_input, input_connection, Arc::clone(connection_change)))]; if let Some(aliases) = &cfg_input.aliases { for alias in aliases { - inputs.push(ProviderConfigWrapper::new(ProviderConfig::new_alias(cfg_input, alias, get_connection, Arc::clone(connection_change)))); + let alias_connection = get_or_create_provider_connection(provider_connections, alias.name.as_str()); + inputs.push(ProviderConfigWrapper::new(ProviderConfig::new_alias( + cfg_input, + alias, + alias_connection, + Arc::clone(connection_change), + ))); } } let mut providers = HashMap::new(); @@ -271,7 +278,6 @@ impl MultiProviderLineup { Self { name: cfg_input.name.clone(), providers, - index: AtomicUsize::new(0), } } @@ -312,6 +318,16 @@ impl MultiProviderLineup { } ProviderPriorityGroup::MultiProviderGroup(index, pg) => { let provider_count = pg.len(); + if grace { + for p in pg { + let result = p.try_allocate(true, grace_period_timeout_secs).await; + if !matches!(result, ProviderAllocation::Exhausted) { + return result; + } + } + return ProviderAllocation::Exhausted; + } + let start = index.fetch_add(1, Ordering::AcqRel) % provider_count; let mut idx = start; @@ -396,72 +412,50 @@ impl MultiProviderLineup { /// } /// ``` async fn acquire(&self, with_grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { - let provider_count = self.providers.len(); - let start = self.index.fetch_add(1, Ordering::AcqRel) % provider_count; - let mut idx = start; - - loop { - let priority_group = &self.providers[idx]; - let allocation = { - let without_grace_allocation = Self::acquire_next_provider_from_group(priority_group, false, grace_period_timeout_secs).await; - - if with_grace && matches!(without_grace_allocation, ProviderAllocation::Exhausted) { - Self::acquire_next_provider_from_group(priority_group, true, grace_period_timeout_secs).await - } else { - without_grace_allocation - } - }; - + // Phase 1: prefer providers with available capacity (no grace allocations), + // scanning priority groups from highest -> lowest. + for priority_group in &self.providers { + let allocation = + Self::acquire_next_provider_from_group(priority_group, false, grace_period_timeout_secs).await; if !matches!(allocation, ProviderAllocation::Exhausted) { - if priority_group.is_exhausted().await { - self.index.store((idx + 1) % provider_count, Ordering::Release); - } return allocation; } - - idx = (idx + 1) % provider_count; - - // loop end - if idx == start { - break; - } } + if !with_grace { + return ProviderAllocation::Exhausted; + } + + // Phase 2: all providers are at capacity, allow grace allocations (still respecting priority order). + for priority_group in &self.providers { + let allocation = + Self::acquire_next_provider_from_group(priority_group, true, grace_period_timeout_secs).await; + if !matches!(allocation, ProviderAllocation::Exhausted) { + return allocation; + } + } ProviderAllocation::Exhausted } // it intended to use with redirects to cycle through provider async fn get_next(&self, grace_period_timeout_secs: u64) -> Option> { - let provider_count = self.providers.len(); - - let start = self.index.fetch_add(1, Ordering::AcqRel) % provider_count; - let mut idx = start; - - loop { - let priority_group = &self.providers[idx]; - - let allocation = { - let config = Self::get_next_provider_from_group(priority_group, false, grace_period_timeout_secs).await; - if config.is_none() { - Self::get_next_provider_from_group(priority_group, true, grace_period_timeout_secs).await - } else { - config - } - }; - - if let Some(config) = allocation { - if priority_group.is_exhausted().await { - self.index.store((idx + 1) % provider_count, Ordering::Release); - } + // Phase 1: prefer providers with available capacity (no grace allocations), + // scanning priority groups from highest -> lowest. + for priority_group in &self.providers { + if let Some(config) = + Self::get_next_provider_from_group(priority_group, false, grace_period_timeout_secs).await + { return Some(config); } + } - idx = (idx + 1) % provider_count; - - // loop end - if idx == start { - break; + // Phase 2: no provider is available, allow grace. + for priority_group in &self.providers { + if let Some(config) = + Self::get_next_provider_from_group(priority_group, true, grace_period_timeout_secs).await + { + return Some(config); } } @@ -510,40 +504,47 @@ impl MultiProviderLineup { } } - pub(in crate::api::model) struct ProviderLineupManager { grace_period_millis: AtomicU64, grace_period_timeout_secs: AtomicU64, inputs: Arc>>>, providers: Arc>>, + provider_connections: DashMap>>, event_manager: Arc, } impl ProviderLineupManager { pub fn new(inputs: Vec>, grace_period_millis: u64, grace_period_timeout_secs: u64, event_manager: &Arc) -> Self { - let lineups = inputs.iter().map(|i| Self::create_lineup(i, None, event_manager)).collect(); + let provider_connections: DashMap>> = DashMap::new(); + let lineups = inputs + .iter() + .map(|i| Self::create_lineup(i, &provider_connections, event_manager)) + .collect(); Self { grace_period_millis: AtomicU64::new(grace_period_millis), grace_period_timeout_secs: AtomicU64::new(grace_period_timeout_secs), inputs: Arc::new(ArcSwap::from_pointee(inputs)), providers: Arc::new(ArcSwap::from_pointee(lineups)), + provider_connections, event_manager: Arc::clone(event_manager), } } - fn create_lineup(cfg_input: &ConfigInput, provider_connections: Option<&HashMap<&str, ProviderConfigConnection>>, event_manager: &Arc) -> ProviderLineup { - let get_connections = provider_connections.map(|c| |name: &str| c.get(name)); - - //let cfg_name = cfg_input.name.clone(); + fn create_lineup( + cfg_input: &ConfigInput, + provider_connections: &DashMap>>, + event_manager: &Arc, + ) -> ProviderLineup { let event_manager = Arc::clone(event_manager); let on_connection_change: ProviderConnectionChangeCallback = Arc::new(move |name: &str, connections: usize| { event_manager.send_provider_event(name, connections); }); if cfg_input.aliases.as_ref().is_some_and(|a| !a.is_empty()) { - ProviderLineup::Multi(MultiProviderLineup::new(cfg_input, get_connections, &on_connection_change)) + ProviderLineup::Multi(MultiProviderLineup::new(cfg_input, provider_connections, &on_connection_change)) } else { - ProviderLineup::Single(SingleProviderLineup::new(cfg_input, get_connections, &on_connection_change)) + let connection = get_or_create_provider_connection(provider_connections, cfg_input.name.as_str()); + ProviderLineup::Single(SingleProviderLineup::new(cfg_input, connection, &on_connection_change)) } } @@ -607,7 +608,7 @@ impl ProviderLineupManager { false } - pub async fn update_config(&self, new_inputs: Vec>, grace_period_millis: u64, grace_period_timeout_secs: u64) { + pub fn update_config(&self, new_inputs: Vec>, grace_period_millis: u64, grace_period_timeout_secs: u64) { self.grace_period_millis.store(grace_period_millis, Ordering::Relaxed); self.grace_period_timeout_secs.store(grace_period_timeout_secs, Ordering::Relaxed); @@ -615,34 +616,9 @@ impl ProviderLineupManager { return; } - let old_lineups = self.providers.load(); - let mut provider_connections = HashMap::new(); - for lineup in old_lineups.iter() { - match lineup { - ProviderLineup::Single(single) => { - provider_connections.insert(single.provider.name.as_str(), single.provider.get_connection_info().await); - } - ProviderLineup::Multi(multi) => { - for group in &multi.providers { - match group { - ProviderPriorityGroup::SingleProviderGroup(cfg) => { - provider_connections.insert(cfg.name.as_str(), cfg.get_connection_info().await); - } - ProviderPriorityGroup::MultiProviderGroup(_, cfgs) => { - for cfg in cfgs { - provider_connections.insert(cfg.name.as_str(), cfg.get_connection_info().await); - } - } - } - } - } - } - } - let mut new_lineups: Vec = Vec::with_capacity(new_inputs.len()); - let connections = Some(provider_connections); for input in &new_inputs { - new_lineups.push(Self::create_lineup(input, connections.as_ref(), &self.event_manager)); + new_lineups.push(Self::create_lineup(input, &self.provider_connections, &self.event_manager)); } debug_if_enabled!("inputs {}", sanitize_sensitive_info(&display_vec(&new_inputs))); @@ -656,13 +632,17 @@ impl ProviderLineupManager { fn log_allocation(allocation: &ProviderAllocation) { - if log_enabled!(log::Level::Debug) { - match allocation { - ProviderAllocation::Exhausted => {} - ProviderAllocation::Available(ref cfg) | - ProviderAllocation::GracePeriod(ref cfg) => { - debug!("Using provider {}", cfg.name); - } + match allocation { + ProviderAllocation::Exhausted => {} + ProviderAllocation::Available(ref cfg) | ProviderAllocation::GracePeriod(ref cfg) => { + debug_if_enabled!( + "Using provider {} (pool user: {}, max_connections: {})", + sanitize_sensitive_info(&cfg.name), + cfg.get_user_info() + .map_or_else(|| "?".to_string(), |ui| sanitize_sensitive_info(&ui.username).to_string()) + , + cfg.max_connections() + ); } } } @@ -679,16 +659,78 @@ impl ProviderLineupManager { // Returns the next available provider connection pub(crate) async fn acquire_connection(&self, input_name: &str) -> ProviderAllocation { + self.acquire_connection_with_grace_override(input_name, true).await + } + + /// Acquire a provider connection, optionally allowing provider-side grace allocations. + /// + /// When `allow_grace` is `false`, the lineup will not allocate providers in `GracePeriod`, + /// even if a global grace period is configured. + pub(crate) async fn acquire_connection_with_grace_override( + &self, + input_name: &str, + allow_grace: bool, + ) -> ProviderAllocation { let providers = self.providers.load(); - let allocation = match Self::get_provider_config_by_name(input_name, &providers) { + let lineup_opt = Self::get_provider_config_by_name(input_name, &providers); + let with_grace = allow_grace && self.grace_period_millis.load(Ordering::Acquire) > 0; + let allocation = match lineup_opt { None => ProviderAllocation::Exhausted, // No Name matched, we don't have this provider - Some((lineup, _config)) => lineup.acquire(self.grace_period_millis.load(Ordering::Acquire) > 0, - self.grace_period_timeout_secs.load(Ordering::Acquire)).await + Some((lineup, _config)) => { + lineup + .acquire(with_grace, self.grace_period_timeout_secs.load(Ordering::Acquire)) + .await + } }; + if matches!(allocation, ProviderAllocation::Exhausted) { + if let Some((lineup, _cfg)) = lineup_opt { + Self::log_exhausted_pool_snapshot(input_name, lineup).await; + } + } Self::log_allocation(&allocation); allocation } + async fn log_exhausted_pool_snapshot(input_name: &str, lineup: &ProviderLineup) { + if !log_enabled!(log::Level::Debug) { + return; + } + + let mut entries: Vec = Vec::new(); + match lineup { + ProviderLineup::Single(single) => { + entries.push(Self::format_provider_snapshot_entry(&single.provider).await); + } + ProviderLineup::Multi(multi) => { + for group in &multi.providers { + match group { + ProviderPriorityGroup::SingleProviderGroup(cfg) => { + entries.push(Self::format_provider_snapshot_entry(cfg).await); + } + ProviderPriorityGroup::MultiProviderGroup(_, cfgs) => { + for cfg in cfgs { + entries.push(Self::format_provider_snapshot_entry(cfg).await); + } + } + } + } + } + } + + debug_if_enabled!( + "Provider pool exhausted for input {} (pool_snapshot=[{}])", + sanitize_sensitive_info(input_name), + sanitize_sensitive_info(&entries.join(", ")) + ); + } + + async fn format_provider_snapshot_entry(cfg: &ProviderConfigWrapper) -> String { + let current = cfg.get_current_connections().await; + let max = cfg.max_connections(); + let expired = is_input_expired(cfg.exp_date()); + format!("{}:{}/{} expired={expired}", cfg.name, current, max) + } + // This method is used for redirects to cycle through provider // pub async fn get_next_provider(&self, input_name: &str) -> Option> { @@ -823,6 +865,7 @@ mod tests { staged: None, exp_date: None, t_batch_url: None, + panel_api: None, } } @@ -853,9 +896,9 @@ mod tests { input.aliases = Some(vec![alias]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; + let provider_connections: DashMap>> = DashMap::new(); // Create MultiProviderLineup with the provider and alias - let lineup = MultiProviderLineup::new(&input, Some(dummy_get_connection), &change_callback); + let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { // Test that the alias provider is available @@ -878,8 +921,8 @@ mod tests { // Adding alias with different priority input.aliases = Some(vec![alias]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; - let lineup = MultiProviderLineup::new(&input, Some(dummy_get_connection), &change_callback); + let provider_connections: DashMap>> = DashMap::new(); + let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); // The alias has a higher priority, so the alias should be acquired first let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -900,8 +943,8 @@ mod tests { // Adding multiple aliases input.aliases = Some(vec![alias1, alias2]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; - let lineup = MultiProviderLineup::new(&input, Some(dummy_get_connection), &change_callback); + let provider_connections: DashMap>> = DashMap::new(); + let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { // The alias with priority 0 should be acquired first (higher priority) @@ -931,8 +974,8 @@ mod tests { // Adding alias input.aliases = Some(vec![alias1, alias2]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; - let lineup = MultiProviderLineup::new(&input, Some(dummy_get_connection), &change_callback); + let provider_connections: DashMap>> = DashMap::new(); + let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { // Acquire connection from alias2 @@ -959,8 +1002,9 @@ mod tests { fn test_acquire_when_capacity_available() { let cfg = create_config_input(1, "provider5_1", 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; - let lineup = SingleProviderLineup::new(&cfg, Some(dummy_get_connection), &change_callback); + let provider_connections: DashMap>> = DashMap::new(); + let connection = get_or_create_provider_connection(&provider_connections, cfg.name.as_str()); + let lineup = SingleProviderLineup::new(&cfg, connection, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { // First acquire attempt should succeed @@ -980,9 +1024,9 @@ mod tests { fn test_release_connection() { let cfg = create_config_input(1, "provider7_1", 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; - - let lineup = SingleProviderLineup::new(&cfg, Some(dummy_get_connection), &change_callback); + let provider_connections: DashMap>> = DashMap::new(); + let connection = get_or_create_provider_connection(&provider_connections, cfg.name.as_str()); + let lineup = SingleProviderLineup::new(&cfg, connection, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { // Acquire two connections @@ -1011,8 +1055,8 @@ mod tests { // Create MultiProviderLineup with the provider and alias let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; - let lineup = MultiProviderLineup::new(&cfg1, Some(dummy_get_connection), &change_callback); + let provider_connections: DashMap>> = DashMap::new(); + let lineup = MultiProviderLineup::new(&cfg1, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { // Test acquiring the first provider @@ -1044,8 +1088,9 @@ mod tests { fn test_concurrent_acquire() { let cfg = create_config_input(1, "provider9_1", 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let dummy_get_connection = |_s: &str| -> Option<&ProviderConfigConnection> { None }; - let lineup = Arc::new(SingleProviderLineup::new(&cfg, Some(dummy_get_connection), &change_callback)); + let provider_connections: DashMap>> = DashMap::new(); + let connection = get_or_create_provider_connection(&provider_connections, cfg.name.as_str()); + let lineup = Arc::new(SingleProviderLineup::new(&cfg, connection, &change_callback)); let available_count = Arc::new(AtomicU16::new(2)); let grace_period_count = Arc::new(AtomicU16::new(1)); diff --git a/backend/src/api/model/streams/active_client_stream.rs b/backend/src/api/model/streams/active_client_stream.rs index fd6ed6b0c..49a5e06cb 100644 --- a/backend/src/api/model/streams/active_client_stream.rs +++ b/backend/src/api/model/streams/active_client_stream.rs @@ -1,49 +1,52 @@ -use crate::api::model::{AppState, ConnectionManager, CustomVideoStreamType, ProviderHandle, StreamDetails}; use crate::api::model::BoxedProviderStream; use crate::api::model::StreamError; use crate::api::model::TimedClientStream; use crate::api::model::TransportStreamBuffer; +use crate::api::model::{AppState, ConnectionManager, CustomVideoStreamType, ProviderHandle, StreamDetails}; +use crate::auth::Fingerprint; use crate::model::ProxyUserCredentials; +use crate::utils::debug_if_enabled; +use axum::http::header::USER_AGENT; +use axum::http::HeaderMap; use bytes::Bytes; +use futures::task::AtomicWaker; use futures::Stream; use futures::StreamExt; use log::{error, info}; use shared::model::{StreamChannel, UserConnectionPermission}; +use shared::utils::sanitize_sensitive_info; use std::pin::Pin; use std::sync::atomic::AtomicU8; -use std::sync::{Arc}; -use std::task::{Poll}; -use axum::http::header::USER_AGENT; -use axum::http::HeaderMap; -use futures::task::AtomicWaker; -use crate::auth::Fingerprint; +use std::sync::Arc; +use std::task::Poll; const INNER_STREAM: u8 = 0_u8; -const GRACE_BLOCK_STREAM: u8 = 1_u8; -const USER_EXHAUSTED_STREAM: u8 = 2_u8; -const PROVIDER_EXHAUSTED_STREAM: u8 = 3_u8; +const USER_EXHAUSTED_STREAM: u8 = 1_u8; +const PROVIDER_EXHAUSTED_STREAM: u8 = 2_u8; +const CHANNEL_UNAVAILABLE_STREAM: u8 = 3_u8; pub(in crate::api) struct ActiveClientStream { inner: BoxedProviderStream, send_custom_stream_flag: Option>, #[allow(dead_code)] provider_handle: Option, - custom_video: (Option, Option), + custom_video: (Option, Option, Option), waker: Option>, connection_manager: Arc, + fingerprint: Arc, + provider_stopped: bool, } impl ActiveClientStream { - #[allow(clippy::too_many_arguments)] pub(crate) async fn new(mut stream_details: StreamDetails, - app_state: &Arc, - user: &ProxyUserCredentials, - connection_permission: UserConnectionPermission, - fingerprint: &Fingerprint, - stream_channel: StreamChannel, - session_token: Option<&str>, - req_headers: &HeaderMap) -> Self { + app_state: &Arc, + user: &ProxyUserCredentials, + connection_permission: UserConnectionPermission, + fingerprint: &Fingerprint, + stream_channel: StreamChannel, + session_token: Option<&str>, + req_headers: &HeaderMap) -> Self { if connection_permission == UserConnectionPermission::Exhausted { error!("Something is wrong this should not happen"); } @@ -55,19 +58,26 @@ impl ActiveClientStream { let virtual_id = stream_channel.virtual_id; app_state.connection_manager.update_connection(username, user.max_connections, fingerprint, &provider_name, stream_channel, user_agent, session_token).await; - if let Some((_,_,_m_, Some(cvt))) = stream_details.stream_info.as_ref() { + if let Some((_, _, _m_, Some(cvt))) = stream_details.stream_info.as_ref() { app_state.connection_manager.update_stream_detail(&fingerprint.addr, *cvt).await; } let cfg = &app_state.app_config; - let waker = Some(Arc::new(AtomicWaker::new())); - let waker_clone = waker.clone(); - let grace_stop_flag = Self::stream_grace_period(app_state, &stream_details, grant_user_grace_period, user, fingerprint, waker_clone.clone()); + let (grace_stop_flag, waker) = if grant_user_grace_period || (stream_details.has_grace_period() && stream_details.provider_name.is_some()) { + let waker = Arc::new(AtomicWaker::new()); + let flag = Self::stream_grace_period(app_state, &stream_details, grant_user_grace_period, user, fingerprint, Some(Arc::clone(&waker))); + let maybe_waker = flag.as_ref().map(|_| waker); + (flag, maybe_waker) + } else { + (Self::stream_grace_period(app_state, &stream_details, grant_user_grace_period, user, fingerprint, None), None) + }; + let custom_response = cfg.custom_stream_response.load(); let custom_video = custom_response.as_ref() - .map_or((None, None), |c| + .map_or((None, None, None), |c| ( c.user_connections_exhausted.clone(), - c.provider_connections_exhausted.clone() + c.provider_connections_exhausted.clone(), + c.channel_unavailable.clone(), )); let stream = match stream_details.stream.take() { @@ -98,7 +108,9 @@ impl ActiveClientStream { send_custom_stream_flag: grace_stop_flag, custom_video, waker, - connection_manager: Arc::clone(&app_state.connection_manager) + connection_manager: Arc::clone(&app_state.connection_manager), + fingerprint: Arc::new(fingerprint.clone()), + provider_stopped: false, } } @@ -128,11 +140,10 @@ impl ActiveClientStream { }; if provider_grace_check.is_some() || user_grace_check.is_some() { - let stream_strategy_flag = Arc::new(AtomicU8::new(GRACE_BLOCK_STREAM)); + let stream_strategy_flag = Arc::new(AtomicU8::new(INNER_STREAM)); let stream_strategy_flag_copy = Arc::clone(&stream_strategy_flag); let grace_period_millis = stream_details.grace_period_millis; - let address = fingerprint.addr; let user_manager = Arc::clone(&active_users); let provider_manager = Arc::clone(&active_provider); let connection_manager = Arc::clone(&connection_manager); @@ -168,9 +179,8 @@ impl ActiveClientStream { } if updated { - connection_manager.release_provider_connection(&address).await; if let Some(flag) = reconnect_flag { - flag.notify(); + flag.notify(); } } @@ -182,11 +192,37 @@ impl ActiveClientStream { } None } + + fn stop_provider_stream(&mut self, unavailable: bool) { + self.provider_stopped = true; + if self.provider_handle.is_some() { + let mgr = Arc::clone(&self.connection_manager); + let handle = self.provider_handle.take(); + + if unavailable { + if let Some(flag) = &self.send_custom_stream_flag { + flag.store(CHANNEL_UNAVAILABLE_STREAM, std::sync::atomic::Ordering::Release); + } + } + + let con_man = Arc::clone(&self.connection_manager); + let addr = self.fingerprint.addr; + self.inner = futures::stream::empty::>().boxed(); + tokio::spawn(async move { + con_man.update_stream_detail(&addr, if unavailable { CustomVideoStreamType::ChannelUnavailable } else { CustomVideoStreamType::UserConnectionsExhausted }).await; + debug_if_enabled!("Provider stream stopped due to grace period or unavailable provider channel for {}", sanitize_sensitive_info(&addr.to_string())); + mgr.release_provider_handle(handle).await; + }); + } + } } impl Stream for ActiveClientStream { type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { + if let Some(waker) = &self.waker { + waker.register(cx.waker()); + } let flag = { match &self.send_custom_stream_flag { Some(flag) => flag.load(std::sync::atomic::Ordering::Acquire), @@ -194,15 +230,22 @@ impl Stream for ActiveClientStream { } }; - if flag == GRACE_BLOCK_STREAM { - if let Some(waker) = &self.waker { - waker.register(cx.waker()); - } - return Poll::Pending; - } - if flag == INNER_STREAM { - return Pin::new(&mut self.inner).poll_next(cx); + match Pin::new(&mut self.inner).poll_next(cx) { + Poll::Ready(Some(Err(e))) => { + error!("Inner stream error: {e:?}"); + self.stop_provider_stream(true); + return Poll::Ready(Some(Err(e))); + } + Poll::Ready(None) => { + self.stop_provider_stream(true); + return Poll::Ready(None); + } + healthy => return healthy, + } + } + if !self.provider_stopped { + self.stop_provider_stream(false); } let buffer_opt = match flag { @@ -212,14 +255,18 @@ impl Stream for ActiveClientStream { PROVIDER_EXHAUSTED_STREAM => { self.custom_video.1.as_mut() } + CHANNEL_UNAVAILABLE_STREAM => { + self.custom_video.2.as_mut() + } _ => None, }; if let Some(buffer) = buffer_opt { - return Poll::Ready(Some(Ok(buffer.next_chunk()))); + Poll::Ready(Some(Ok(buffer.next_chunk()))) + } else { + // At this point it should be the empty stream + Pin::new(&mut self.inner).poll_next(cx) } - - Poll::Ready(None) } } @@ -231,4 +278,4 @@ impl Drop for ActiveClientStream { mgr.release_provider_handle(hndl).await; }); } -} \ No newline at end of file +} diff --git a/backend/src/api/model/streams/shared_stream_manager.rs b/backend/src/api/model/streams/shared_stream_manager.rs index e3e0f2fc1..6ed204b04 100644 --- a/backend/src/api/model/streams/shared_stream_manager.rs +++ b/backend/src/api/model/streams/shared_stream_manager.rs @@ -430,19 +430,18 @@ impl SharedStreamManager { manager: Arc, ) -> Option<(BoxedProviderStream, Option)> { let shared_state_opt = { - let shared_streams = self.shared_streams.read().await; - shared_streams.by_key.get(stream_url).cloned() + let mut shared_streams = self.shared_streams.write().await; + if let Some(shared_state) = shared_streams.by_key.get(stream_url).cloned() { + shared_streams.key_by_addr.insert(*addr, stream_url.to_owned()); + Some(shared_state) + } else { + None + } }; if let Some(shared_state) = shared_state_opt { - { - let mut shared_streams = self.shared_streams.write().await; - shared_streams.key_by_addr.insert(*addr, stream_url.to_owned()); - } - debug_if_enabled!("Responding to existing shared client stream {} {}", - sanitize_sensitive_info(&addr.to_string()), sanitize_sensitive_info(stream_url) - ); + sanitize_sensitive_info(&addr.to_string()), sanitize_sensitive_info(stream_url)); Some(shared_state.subscribe(addr, manager).await) } else { None diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs new file mode 100644 index 000000000..acc6b5987 --- /dev/null +++ b/backend/src/api/panel_api.rs @@ -0,0 +1,832 @@ +use crate::api::model::AppState; +use crate::model::{ConfigInput, is_input_expired, PanelApiQueryParam, PanelApiConfig}; +use crate::utils::{debug_if_enabled, save_sources_config}; +use crate::utils::{get_csv_file_path}; +use log::{debug, error, warn}; +use serde_json::Value; +use shared::error::{create_tuliprox_error_result, info_err, TuliproxError, TuliproxErrorKind}; +use shared::model::{InputType}; +use shared::utils::{get_credentials_from_url, parse_timestamp, sanitize_sensitive_info, trim_last_slash}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use url::Url; +use crate::api::config_watch::ConfigFile; + +#[derive(Debug, Clone)] +struct AccountCredentials { + name: String, + username: String, + password: String, + exp_date: Option, +} + +fn parse_boolish(value: &Value) -> bool { + match value { + Value::Bool(b) => *b, + Value::Number(n) => n.as_i64().unwrap_or(0) != 0, + Value::String(s) => matches!(s.trim().to_lowercase().as_str(), "true" | "1" | "yes" | "y" | "ok"), + _ => false, + } +} + +fn is_date_only_yyyy_mm_dd(value: &str) -> bool { + let value = value.trim(); + if value.len() != 10 { + return false; + } + let bytes = value.as_bytes(); + bytes[4] == b'-' + && bytes[7] == b'-' + && bytes[0..4].iter().all(u8::is_ascii_digit) + && bytes[5..7].iter().all(u8::is_ascii_digit) + && bytes[8..10].iter().all(u8::is_ascii_digit) +} + +fn first_json_object(value: &Value) -> Option<&serde_json::Map> { + match value { + Value::Array(arr) => arr.first().and_then(|v| v.as_object()), + Value::Object(obj) => Some(obj), + _ => None, + } +} + +fn extract_username_password_from_json(obj: &serde_json::Map) -> Option<(String, String)> { + let username = obj.get("username").and_then(|v| v.as_str()).map(str::trim).filter(|s| !s.is_empty()); + let password = obj.get("password").and_then(|v| v.as_str()).map(str::trim).filter(|s| !s.is_empty()); + match (username, password) { + (Some(u), Some(p)) => Some((u.to_string(), p.to_string())), + _ => None, + } +} + +fn extract_username_password_from_url(url_str: &str) -> Option<(String, String)> { + Url::parse(url_str).ok().and_then(|url| { + let (u, p) = get_credentials_from_url(&url); + match (u, p) { + (Some(u), Some(p)) if !u.trim().is_empty() && !p.trim().is_empty() => Some((u, p)), + _ => None, + } + }) +} + +fn extract_base_url(url_str: &str) -> Option { + Url::parse(url_str).ok().map(|u| u.origin().ascii_serialization()) +} + +fn resolve_query_params( + params: &[PanelApiQueryParam], + api_key: Option<&str>, + creds: Option<(&str, &str)>, +) -> Result, TuliproxError> { + let mut out = Vec::with_capacity(params.len()); + for p in params { + let key = p.key.trim(); + if key.is_empty() { + continue; + } + let mut value = p.value.trim().to_string(); + if value.eq_ignore_ascii_case("auto") { + if key.eq_ignore_ascii_case("api_key") { + let Some(k) = api_key.filter(|s| !s.trim().is_empty()) else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but panel_api.api_key is missing"); + }; + value = k.to_string(); + } else if key.eq_ignore_ascii_case("username") { + let Some((u, _)) = creds else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but no account username is available"); + }; + value = u.to_string(); + } else if key.eq_ignore_ascii_case("password") { + let Some((_, pw)) = creds else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but no account password is available"); + }; + value = pw.to_string(); + } + } + out.push((key.to_string(), value)); + } + Ok(out) +} + +fn build_panel_url(base_url: &str, query_params: &[(String, String)]) -> Result { + let mut url = Url::parse(base_url).map_err(|e| info_err!(format!("panel_api: invalid url {base_url}: {e}")))?; + { + let mut pairs = url.query_pairs_mut(); + for (k, v) in query_params { + pairs.append_pair(k, v); + } + } + Ok(url) +} + +fn sanitize_panel_api_json_for_log(value: &Value) -> Value { + match value { + Value::Array(arr) => Value::Array(arr.iter().map(sanitize_panel_api_json_for_log).collect()), + Value::Object(obj) => { + let mut out = serde_json::Map::with_capacity(obj.len()); + for (k, v) in obj { + if k.eq_ignore_ascii_case("api_key") || k.eq_ignore_ascii_case("apikey") || k.eq_ignore_ascii_case("token") { + out.insert(k.clone(), Value::String("***".to_string())); + continue; + } + if k.eq_ignore_ascii_case("username") || k.eq_ignore_ascii_case("password") { + out.insert(k.clone(), Value::String("***".to_string())); + continue; + } + if k.eq_ignore_ascii_case("url") { + if let Some(s) = v.as_str() { + out.insert(k.clone(), Value::String(sanitize_sensitive_info(s).into_owned())); + continue; + } + } + out.insert(k.clone(), sanitize_panel_api_json_for_log(v)); + } + Value::Object(out) + } + other => other.clone(), + } +} + +async fn panel_get_json(app_state: &AppState, url: Url) -> Result { + let client = app_state.http_client.load(); + let sanitized = sanitize_sensitive_info(url.as_str()); + debug_if_enabled!("panel_api request {}", sanitized); + let resp = client + .get(url) + .timeout(std::time::Duration::from_secs(30)) + .send() + .await + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api request failed: {e}")))?; + let status = resp.status(); + let body = resp + .text() + .await + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api read response failed: {e}")))?; + let json: Value = serde_json::from_str(&body) + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api invalid json (http {status}): {e}")))?; + let json_for_log = sanitize_panel_api_json_for_log(&json); + if let Ok(json_str) = serde_json::to_string(&json_for_log) { + debug_if_enabled!("panel_api response (http {}): {}", status, sanitize_sensitive_info(&json_str)); + } + Ok(json) +} + +async fn panel_client_new(app_state: &AppState, cfg: &PanelApiConfig) -> Result<(String, String, Option), TuliproxError> { + let params = resolve_query_params(&cfg.query_parameter.client_new, cfg.api_key.as_deref(), None)?; + let url = build_panel_url(cfg.url.as_str(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new response is not a JSON object/array"); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new status=false"); + } + if let Some((u, p)) = extract_username_password_from_json(obj) { + return Ok((u, p, None)); + } + if let Some(url_str) = obj.get("url").and_then(|v| v.as_str()) { + if let Some((u, p)) = extract_username_password_from_url(url_str) { + let base = extract_base_url(url_str); + return Ok((u, p, base)); + } + } + create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new response missing username/password (and no parsable url)") +} + +async fn panel_client_renew(app_state: &AppState, cfg: &PanelApiConfig, username: &str, password: &str) -> Result<(), TuliproxError> { + let params = resolve_query_params( + &cfg.query_parameter.client_renew, + cfg.api_key.as_deref(), + Some((username, password)), + )?; + let url = build_panel_url(cfg.url.as_str(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_renew response is not a JSON object/array"); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_renew status=false"); + } + Ok(()) +} + +async fn panel_client_info(app_state: &AppState, cfg: &PanelApiConfig, username: &str, password: &str) -> Result, TuliproxError> { + let params = resolve_query_params( + &cfg.query_parameter.client_info, + cfg.api_key.as_deref(), + Some((username, password)), + )?; + let url = build_panel_url(cfg.url.as_str(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_info response is not a JSON object/array"); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_info status=false"); + } + let expire = obj.get("expire").and_then(|v| v.as_str()).unwrap_or_default().trim(); + let parsed = parse_timestamp(expire).ok().flatten(); + if parsed.is_some() { + return Ok(parsed); + } + + // Some panels return only a date ("YYYY-MM-DD") without time. Normalize to midnight. + if is_date_only_yyyy_mm_dd(expire) { + let normalized = format!("{expire} 00:00:00"); + return Ok(parse_timestamp(&normalized).ok().flatten()); + } + + Ok(None) +} + +fn extract_account_creds_from_input(input: &ConfigInput) -> Option<(String, String)> { + if let (Some(u), Some(p)) = (input.username.as_deref(), input.password.as_deref()) { + if !u.trim().is_empty() && !p.trim().is_empty() { + return Some((u.to_string(), p.to_string())); + } + } + Url::parse(input.url.as_str()).ok().and_then(|u| { + let (uu, pp) = get_credentials_from_url(&u); + match (uu, pp) { + (Some(uu), Some(pp)) if !uu.trim().is_empty() && !pp.trim().is_empty() => Some((uu, pp)), + _ => None, + } + }) +} + +fn collect_expired_accounts(input: &ConfigInput) -> Vec { + let mut out = Vec::new(); + if is_input_expired(input.exp_date) { + if let Some((u, p)) = extract_account_creds_from_input(input) { + out.push(AccountCredentials { + name: input.name.clone(), + username: u, + password: p, + exp_date: input.exp_date, + }); + } + } + if let Some(aliases) = input.aliases.as_ref() { + for a in aliases { + if is_input_expired(a.exp_date) { + if let (Some(u), Some(p)) = (a.username.as_deref(), a.password.as_deref()) { + if !u.trim().is_empty() && !p.trim().is_empty() { + out.push(AccountCredentials { + name: a.name.clone(), + username: u.to_string(), + password: p.to_string(), + exp_date: a.exp_date, + }); + } + } + } + } + } + out.sort_by_key(|a| a.exp_date.unwrap_or(i64::MAX)); + out +} + +#[allow(clippy::too_many_arguments)] +async fn patch_source_yml_add_alias( + app_state: &Arc, + source_file_path: &Path, + input_name: &str, + alias_name: &str, + base_url: &str, + username: &str, + password: &str, + exp_date: Option, +) -> Result<(), TuliproxError> { + let raw = tokio::fs::read_to_string(source_file_path) + .await + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read source file: {e}")))?; + let mut doc: serde_yaml::Value = serde_yaml::from_str(&raw) + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to parse source file yaml: {e}")))?; + + let Some(root) = doc.as_mapping_mut() else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: source.yml root is not a mapping"); + }; + let sources = root.get_mut(serde_yaml::Value::String("sources".to_string())).and_then(|v| v.as_sequence_mut()); + let Some(sources) = sources else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: source.yml missing 'sources' list"); + }; + + let mut found_input = None; + for src in sources.iter_mut() { + let Some(src_map) = src.as_mapping_mut() else { continue; }; + let Some(inputs) = src_map.get_mut(serde_yaml::Value::String("inputs".to_string())).and_then(|v| v.as_sequence_mut()) else { continue; }; + for inp in inputs.iter_mut() { + let Some(inp_map) = inp.as_mapping_mut() else { continue; }; + let name = inp_map.get(serde_yaml::Value::String("name".to_string())).and_then(|v| v.as_str()); + if name == Some(input_name) { + found_input = Some(inp_map); + break; + } + } + if found_input.is_some() { + break; + } + } + let Some(inp_map) = found_input else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find input '{input_name}' in source.yml"); + }; + + let aliases_key = serde_yaml::Value::String("aliases".to_string()); + let aliases_value = inp_map + .entry(aliases_key.clone()) + .or_insert_with(|| serde_yaml::Value::Sequence(vec![])); + match aliases_value { + serde_yaml::Value::Sequence(_) => {} + serde_yaml::Value::Null => { + *aliases_value = serde_yaml::Value::Sequence(vec![]); + } + serde_yaml::Value::Mapping(_) => { + // Some users might have a single alias stored as a mapping instead of a list. + // Normalize to a list so we can append safely. + let old = std::mem::replace(aliases_value, serde_yaml::Value::Sequence(vec![])); + if let serde_yaml::Value::Mapping(map) = old { + if let serde_yaml::Value::Sequence(seq) = aliases_value { + seq.push(serde_yaml::Value::Mapping(map)); + } + } + } + _ => { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: input.aliases is not a list in source.yml"); + } + } + let Some(alias_seq) = aliases_value.as_sequence_mut() else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: input.aliases is not a list in source.yml"); + }; + + let mut alias_map = serde_yaml::Mapping::new(); + alias_map.insert(serde_yaml::Value::String("name".to_string()), serde_yaml::Value::String(alias_name.to_string())); + alias_map.insert(serde_yaml::Value::String("url".to_string()), serde_yaml::Value::String(base_url.to_string())); + alias_map.insert(serde_yaml::Value::String("username".to_string()), serde_yaml::Value::String(username.to_string())); + alias_map.insert(serde_yaml::Value::String("password".to_string()), serde_yaml::Value::String(password.to_string())); + alias_map.insert(serde_yaml::Value::String("max_connections".to_string()), serde_yaml::Value::Number(1.into())); + if let Some(ts) = exp_date { + alias_map.insert(serde_yaml::Value::String("exp_date".to_string()), serde_yaml::Value::Number(ts.into())); + } + alias_seq.push(serde_yaml::Value::Mapping(alias_map)); + + persist_source_config(app_state, source_file_path, &doc).await?; + Ok(()) +} + +async fn patch_source_yml_update_exp_date( + app_state: &Arc, + source_file_path: &Path, + input_name: &str, + account_name: &str, + exp_date: i64, +) -> Result<(), TuliproxError> { + let raw = tokio::fs::read_to_string(source_file_path) + .await + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read source file: {e}")))?; + let mut doc: serde_yaml::Value = serde_yaml::from_str(&raw) + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to parse source file yaml: {e}")))?; + let Some(root) = doc.as_mapping_mut() else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: source.yml root is not a mapping"); + }; + let sources = root.get_mut(serde_yaml::Value::String("sources".to_string())).and_then(|v| v.as_sequence_mut()); + let Some(sources) = sources else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: source.yml missing 'sources' list"); + }; + for src in sources.iter_mut() { + let Some(src_map) = src.as_mapping_mut() else { continue; }; + let Some(inputs) = src_map.get_mut(serde_yaml::Value::String("inputs".to_string())).and_then(|v| v.as_sequence_mut()) else { continue; }; + for inp in inputs.iter_mut() { + let Some(inp_map) = inp.as_mapping_mut() else { continue; }; + let name = inp_map.get(serde_yaml::Value::String("name".to_string())).and_then(|v| v.as_str()); + if name != Some(input_name) { + continue; + } + if account_name == input_name { + inp_map.insert(serde_yaml::Value::String("exp_date".to_string()), serde_yaml::Value::Number(exp_date.into())); + inp_map.insert(serde_yaml::Value::String("enabled".to_string()), serde_yaml::Value::Bool(true)); + } else if let Some(aliases_value) = inp_map.get_mut(serde_yaml::Value::String("aliases".to_string())) { + // Normalize aliases to a sequence if the file contains a single mapping. + if matches!(aliases_value, serde_yaml::Value::Mapping(_)) { + let old = std::mem::replace(aliases_value, serde_yaml::Value::Sequence(vec![])); + if let serde_yaml::Value::Mapping(map) = old { + if let serde_yaml::Value::Sequence(seq) = aliases_value { + seq.push(serde_yaml::Value::Mapping(map)); + } + } + } + + if let Some(aliases) = aliases_value.as_sequence_mut() { + for a in aliases.iter_mut() { + let Some(a_map) = a.as_mapping_mut() else { continue; }; + let a_name = a_map.get(serde_yaml::Value::String("name".to_string())).and_then(|v| v.as_str()); + if a_name == Some(account_name) { + a_map.insert(serde_yaml::Value::String("exp_date".to_string()), serde_yaml::Value::Number(exp_date.into())); + } + } + } + } + persist_source_config(app_state, source_file_path, &doc).await?; + return Ok(()); + } + } + create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find account '{account_name}' under input '{input_name}' in source.yml") +} + +async fn persist_source_config(app_state: &Arc, source_file_path: &Path, doc: &serde_yaml::Value) -> Result<(), TuliproxError> { + + let paths = app_state.app_config.paths.load(); + let source_file = paths.sources_file_path.as_str(); + let config = app_state.app_config.config.load(); + let backup_dir = config.get_backup_dir(); + save_sources_config(source_file_path.to_str().unwrap_or(source_file), &backup_dir, &doc).await +} + +async fn patch_batch_csv_append( + csv_path: &Path, + batch_type: InputType, + alias_name: &str, + base_url: &str, + username: &str, + password: &str, + exp_date: Option, +) -> Result<(), TuliproxError> { + let raw = tokio::fs::read_to_string(csv_path).await.unwrap_or_default(); + let mut lines: Vec = raw.lines().map(ToString::to_string).collect(); + let header_line_idx = lines.iter().position(|l| l.trim_start().starts_with('#')); + let header = header_line_idx + .and_then(|idx| lines.get(idx).map(|s| s.trim_start_matches('#').trim().to_string())) + .unwrap_or_else(|| match batch_type { + InputType::XtreamBatch => "name;username;password;url;max_connections;priority;exp_date".to_string(), + _ => "url;max_connections;priority;name;username;password;exp_date".to_string(), + }); + let cols: Vec = header.split(';').map(|s| s.trim().to_lowercase()).collect(); + if header_line_idx.is_none() { + lines.insert(0, format!("#{header}")); + } + let mut record: Vec = vec![String::new(); cols.len()]; + for (i, c) in cols.iter().enumerate() { + record[i] = match c.as_str() { + "name" => alias_name.to_string(), + "username" => username.to_string(), + "password" => password.to_string(), + "url" => { + if batch_type == InputType::M3uBatch { + format!( + "{}/get.php?username={}&password={}&type=m3u_plus", + trim_last_slash(base_url), + username, + password + ) + } else { + base_url.to_string() + } + } + "max_connections" => "1".to_string(), + "priority" => "0".to_string(), + "exp_date" => exp_date.map_or(String::new(), |ts| ts.to_string()), + _ => String::new(), + }; + } + lines.push(record.join(";")); + tokio::fs::write(csv_path, lines.join("\n") + "\n") + .await + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to write csv: {e}")))?; + Ok(()) +} + +async fn patch_batch_csv_update_exp_date( + csv_path: &Path, + account_name: &str, + username: &str, + password: &str, + exp_date: i64, +) -> Result<(), TuliproxError> { + let raw = tokio::fs::read_to_string(csv_path) + .await + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read csv: {e}")))?; + let mut lines: Vec = raw.lines().map(ToString::to_string).collect(); + let header_line_idx = lines.iter().position(|l| l.trim_start().starts_with('#')); + let Some(header_idx) = header_line_idx else { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: csv missing header line"); + }; + let header = lines[header_idx].trim_start_matches('#').trim(); + let cols: Vec = header.split(';').map(|s| s.trim().to_lowercase()).collect(); + let exp_idx = cols.iter().position(|c| c == "exp_date"); + let name_idx = cols.iter().position(|c| c == "name"); + let user_idx = cols.iter().position(|c| c == "username"); + let pass_idx = cols.iter().position(|c| c == "password"); + let url_idx = cols.iter().position(|c| c == "url"); + let Some(exp_idx) = exp_idx else { + debug!("panel_api: csv has no exp_date column; skipping exp_date persistence"); + return Ok(()); + }; + + for i in (header_idx + 1)..lines.len() { + let line = lines[i].trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let mut fields: Vec = line.split(';').map(ToString::to_string).collect(); + fields.resize(cols.len(), String::new()); + + let mut matches = false; + if let Some(n_idx) = name_idx { + if fields.get(n_idx).map(|s| s.trim()) == Some(account_name) { + matches = true; + } + } + if !matches { + if let (Some(u_idx), Some(p_idx)) = (user_idx, pass_idx) { + matches = fields.get(u_idx).map(|s| s.trim()) == Some(username) + && fields.get(p_idx).map(|s| s.trim()) == Some(password); + } else if let Some(u_idx) = url_idx { + if let Some(url_str) = fields.get(u_idx) { + if let Some((u, p)) = extract_username_password_from_url(url_str) { + matches = u == username && p == password; + } + } + } + } + if matches { + fields[exp_idx] = exp_date.to_string(); + lines[i] = fields.join(";"); + tokio::fs::write(csv_path, lines.join("\n") + "\n") + .await + .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to write csv: {e}")))?; + return Ok(()); + } + } + warn!("panel_api: could not find batch csv row for account {account_name}"); + Ok(()) +} + +const MAX_ALIAS_NAME_ATTEMPTS: usize = 1000; + +fn derive_unique_alias_name(existing: &[String], input_name: &str, username: &str) -> String { + let base = format!("{input_name}-{username}"); + if !existing.contains(&base) { + return base; + } + for i in 2..MAX_ALIAS_NAME_ATTEMPTS { + let cand = format!("{base}-{i}"); + if !existing.contains(&cand) { + return cand; + } + } + warn!("derive_unique_alias_name: exhausted {MAX_ALIAS_NAME_ATTEMPTS} attempts for base '{base}'; returning potentially duplicate name"); + base +} + +async fn try_renew_expired_account( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + is_batch: bool, + sources_path: &Path, +) -> bool { + let expired = collect_expired_accounts(input); + for acct in &expired { + match panel_client_renew(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str()).await { + Ok(()) => { + let refreshed_exp = panel_client_info(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str()) + .await + .ok() + .flatten(); + + if let Some(new_exp) = refreshed_exp.or(acct.exp_date) { + if is_batch { + let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); + if let Ok(csv_path) = get_csv_file_path(batch_url) { + let _csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; + if let Err(err) = patch_batch_csv_update_exp_date(&csv_path, &acct.name, &acct.username, &acct.password, new_exp).await { + debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); + } + } + } else { + let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; + if let Err(err) = patch_source_yml_update_exp_date(app_state, sources_path, &input.name, &acct.name, new_exp).await { + debug_if_enabled!("panel_api failed to persist renew exp_date to source.yml: {}", err); + } + } + } + + if let Err(err) = reload_sources(app_state).await { + debug_if_enabled!("panel_api reload sources failed: {}", err); + } + return true; + } + Err(err) => { + debug_if_enabled!( + "panel_api client_renew failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(err.to_string().as_str()) + ); + } + } + } + false +} + +async fn try_create_new_account( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + is_batch: bool, + sources_path: &Path, +) -> bool { + match panel_client_new(app_state, panel_cfg).await { + Ok((username, password, base_url_from_resp)) => { + let base_url = base_url_from_resp.unwrap_or_else(|| input.url.clone()); + let base_url = extract_base_url(base_url.as_str()).unwrap_or_else(|| base_url.clone()); + + let mut existing_names: Vec = vec![input.name.clone()]; + if let Some(aliases) = input.aliases.as_ref() { + existing_names.extend(aliases.iter().map(|a| a.name.clone())); + } + let alias_name = derive_unique_alias_name(&existing_names, &input.name, &username); + + let exp_date = panel_client_info(app_state, panel_cfg, &username, &password) + .await + .ok() + .flatten(); + + if is_batch { + let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); + match get_csv_file_path(batch_url) { + Ok(csv_path) => { + let batch_type = if input.input_type == InputType::Xtream { + InputType::XtreamBatch + } else { + InputType::M3uBatch + }; + let _csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; + if let Err(err) = + patch_batch_csv_append(&csv_path, batch_type, &alias_name, &base_url, &username, &password, exp_date).await + { + warn!("panel_api failed to append new account to csv: {err}"); + return false; + } + } + Err(err) => { + warn!( + "panel_api cannot resolve batch csv path {}: {}", + sanitize_sensitive_info(batch_url), + err + ); + return false; + } + } + } else { + let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; + if let Err(err) = + patch_source_yml_add_alias(app_state, sources_path, &input.name, &alias_name, &base_url, &username, &password, exp_date).await + { + warn!("panel_api failed to persist new alias to source.yml: {err}"); + return false; + } + } + + if let Err(err) = reload_sources(app_state).await { + error!("panel_api reload sources failed: {err}"); + return false; + } + true + } + Err(err) => { + debug_if_enabled!("panel_api client_new failed: {}", sanitize_sensitive_info(err.to_string().as_str())); + false + } + } +} + +pub async fn try_provision_account_on_exhausted(app_state: &Arc, input: &ConfigInput) -> bool { + let Some(panel_cfg) = input.panel_api.as_ref() else { + debug_if_enabled!("panel_api: skipped (no panel_api config) for input {}", sanitize_sensitive_info(&input.name)); + return false; + }; + if !panel_cfg.enabled { + debug_if_enabled!("panel_api: skipped (panel_api.enabled false) for input {}", sanitize_sensitive_info(&input.name)); + return false; + } + + if panel_cfg.url.trim().is_empty() { + debug_if_enabled!("panel_api: skipped (panel_api.url empty) for input {}", sanitize_sensitive_info(&input.name)); + return false; + } + + let _input_lock = app_state + .app_config + .file_locks + .write_lock_str(format!("panel_api:{}", input.name).as_str()) + .await; + + debug_if_enabled!( + "panel_api: exhausted -> provisioning for input {} (aliases={})", + sanitize_sensitive_info(&input.name), + input.aliases.as_ref().map_or(0, Vec::len) + ); + + let is_batch = input.t_batch_url.as_ref().is_some_and(|u| !u.trim().is_empty()); + let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); + let sources_path = PathBuf::from(&sources_file_path); + + if try_renew_expired_account(app_state, input, panel_cfg, is_batch, sources_path.as_path()).await { + debug_if_enabled!("panel_api: provisioning succeeded via client_renew for input {}", sanitize_sensitive_info(&input.name)); + return true; + } + let created = try_create_new_account(app_state, input, panel_cfg, is_batch, sources_path.as_path()).await; + debug_if_enabled!( + "panel_api: provisioning via client_new for input {} => {}", + sanitize_sensitive_info(&input.name), + if created { "success" } else { "failed" } + ); + created +} + +pub(crate) async fn sync_panel_api_exp_dates_on_boot(app_state: &Arc) { + let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); + let sources_path = PathBuf::from(&sources_file_path); + let mut any_change = false; + + let sources = app_state.app_config.sources.load(); + for source in &sources.sources { + for input in &source.inputs { + let Some(panel_cfg) = input.panel_api.as_ref() else { continue; }; + if !panel_cfg.enabled || panel_cfg.url.trim().is_empty() { + continue; + } + + let is_batch = input.t_batch_url.as_ref().is_some_and(|u| !u.trim().is_empty()); + let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); + let csv_path = if is_batch { get_csv_file_path(batch_url).ok() } else { None }; + + let mut accounts: Vec = vec![]; + if let Some((u, p)) = extract_account_creds_from_input(input.as_ref()) { + accounts.push(AccountCredentials { + name: input.name.clone(), + username: u, + password: p, + exp_date: input.exp_date, + }); + } + if let Some(aliases) = input.aliases.as_ref() { + for a in aliases { + if let (Some(u), Some(p)) = (a.username.as_deref(), a.password.as_deref()) { + accounts.push(AccountCredentials { + name: a.name.clone(), + username: u.to_string(), + password: p.to_string(), + exp_date: a.exp_date, + }); + } + } + } + + for acct in &accounts { + let new_exp = match panel_client_info(app_state.as_ref(), panel_cfg, &acct.username, &acct.password).await { + Ok(v) => v, + Err(err) => { + debug_if_enabled!( + "panel_api client_info failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(err.to_string().as_str()) + ); + None + } + }; + let Some(new_exp) = new_exp else { continue; }; + if acct.exp_date == Some(new_exp) { + continue; + } + + if let Some(csv_path) = csv_path.as_ref() { + let _csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = patch_batch_csv_update_exp_date(csv_path, &acct.name, &acct.username, &acct.password, new_exp).await { + debug_if_enabled!("panel_api boot sync failed to persist exp_date to csv: {}", err); + continue; + } + } else { + let _src_lock = app_state.app_config.file_locks.write_lock(&sources_path).await; + if let Err(err) = patch_source_yml_update_exp_date(app_state, &sources_path, &input.name, &acct.name, new_exp).await { + debug_if_enabled!("panel_api boot sync failed to persist exp_date to source.yml: {}", err); + continue; + } + } + any_change = true; + } + } + } + + if any_change { + if let Err(err) = reload_sources(app_state).await { + debug_if_enabled!("panel_api boot sync reload sources failed: {}", err); + } + } +} + +async fn reload_sources(app_state: &Arc) -> Result<(), TuliproxError> { + ConfigFile::load_sources(app_state).await +} \ No newline at end of file diff --git a/backend/src/library/classifier.rs b/backend/src/library/classifier.rs index ed73c5e47..a5af44031 100644 --- a/backend/src/library/classifier.rs +++ b/backend/src/library/classifier.rs @@ -8,7 +8,7 @@ use crate::library::{SeriesEpisodeFile, SeriesKey}; fn clear_filename(file_name: &str) -> String { // Remove quality indicators (1080p, 720p, BluRay, etc.) - let mut cleaned = CONSTANTS.re_classifier_quality.replace(file_name, "").to_string(); + let mut cleaned = CONSTANTS.re_quality.replace(file_name, "").to_string(); // Clean up special characters cleaned = cleaned.replace(['.', '_', '-'], " ").trim().to_string(); diff --git a/backend/src/main.rs b/backend/src/main.rs index 9f36d49fe..d2ddaf924 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -10,7 +10,9 @@ mod modules; include_modules!(); use crate::auth::generate_password; -use crate::model::{AppConfig, Config, Healthcheck, HealthcheckConfig, ProcessTargets, SourcesConfig}; +use crate::model::{ + AppConfig, Config, Healthcheck, HealthcheckConfig, ProcessTargets, SourcesConfig, +}; use crate::processing::processor::playlist; use crate::utils::init_logger; use crate::utils::request::create_client; @@ -57,7 +59,12 @@ struct Args { api_proxy: Option, /// Run in server mode - #[arg(short = 's', long, default_value_t = false, default_missing_value = "true")] + #[arg( + short = 's', + long, + default_value_t = false, + default_missing_value = "true" + )] server: bool, /// log level @@ -80,7 +87,6 @@ struct Args { force_library_rescan: bool, } - const VERSION: &str = env!("CARGO_PKG_VERSION"); const BUILD_TIMESTAMP: &str = env!("VERGEN_BUILD_TIMESTAMP"); @@ -106,7 +112,10 @@ async fn main() { let mut config_paths = get_file_paths(&args); - init_logger(args.log_level.as_ref(), config_paths.config_file_path.as_str()); + init_logger( + args.log_level.as_ref(), + config_paths.config_file_path.as_str(), + ); if args.healthcheck { let healthy = healthcheck(config_paths.config_file_path.as_str()).await; @@ -122,14 +131,23 @@ async fn main() { } info!("Version: {VERSION}"); - if let Some(bts) = BUILD_TIMESTAMP.to_string().parse::>().ok().map(|datetime| datetime.format("%Y-%m-%d %H:%M:%S %Z").to_string()) { + if let Some(bts) = BUILD_TIMESTAMP + .to_string() + .parse::>() + .ok() + .map(|datetime| datetime.format("%Y-%m-%d %H:%M:%S %Z").to_string()) + { info!("Build time: {bts}"); } - let app_config = utils::read_initial_app_config(&mut config_paths, true, true, args.server).await.unwrap_or_else(|err| exit!("{}", err)); + let app_config = utils::read_initial_app_config(&mut config_paths, true, true, args.server) + .await + .unwrap_or_else(|err| exit!("{}", err)); print_info(&app_config); let sources = > as Access>::load(&app_config.sources); - let targets = sources.validate_targets(args.target.as_ref()).unwrap_or_else(|err| exit!("{}", err)); + let targets = sources + .validate_targets(args.target.as_ref()) + .unwrap_or_else(|err| exit!("{}", err)); if args.server { start_in_server_mode(Arc::new(app_config), Arc::new(targets)).await; @@ -141,14 +159,23 @@ async fn main() { fn print_info(app_config: &AppConfig) { let config = > as Access>::load(&app_config.config); let paths = > as Access>::load(&app_config.paths); - info!("Current time: {}", chrono::offset::Local::now().format("%Y-%m-%d %H:%M:%S")); + info!( + "Current time: {}", + chrono::offset::Local::now().format("%Y-%m-%d %H:%M:%S") + ); info!("Temp dir: {}", tempfile::env::temp_dir().display()); info!("Working dir: {:?}", &config.working_dir); info!("Config dir: {:?}", &paths.config_path); info!("Config file: {:?}", &paths.config_file_path); info!("Source file: {:?}", &paths.sources_file_path); info!("Api Proxy File: {:?}", &paths.api_proxy_file_path); - info!("Mapping file: {:?}", &paths.mapping_file_path.as_ref().map_or_else(|| "not used", |v| v.as_str())); + info!( + "Mapping file: {:?}", + &paths + .mapping_file_path + .as_ref() + .map_or_else(|| "not used", |v| v.as_str()) + ); if let Some(cache) = config.reverse_proxy.as_ref().and_then(|r| r.cache.as_ref()) { if cache.enabled { diff --git a/backend/src/model/config/input.rs b/backend/src/model/config/input.rs index e2d8a1040..0d34c57b9 100644 --- a/backend/src/model/config/input.rs +++ b/backend/src/model/config/input.rs @@ -11,6 +11,7 @@ use std::collections::HashMap; use std::fmt; use std::path::PathBuf; use url::Url; +use crate::model::config::panel_api::PanelApiConfig; #[allow(clippy::struct_excessive_bools)] #[derive(Debug, Clone)] @@ -142,6 +143,7 @@ pub struct ConfigInput { pub staged: Option, pub exp_date: Option, pub t_batch_url: Option, + pub panel_api: Option, } impl ConfigInput { @@ -161,6 +163,10 @@ impl ConfigInput { self.enabled = false; } + if let Some(panel_api) = &mut self.panel_api { + panel_api.prepare()?; + } + Ok(batch_file_path) } @@ -242,6 +248,7 @@ impl ConfigInput { staged: None, exp_date: None, t_batch_url: None, + panel_api: self.panel_api.clone(), } } } @@ -268,6 +275,7 @@ impl From<&ConfigInputDto> for ConfigInput { exp_date: dto.exp_date, staged: dto.staged.as_ref().map(StagedInput::from), t_batch_url: None, + panel_api: dto.panel_api.as_ref().map(PanelApiConfig::from), } } } @@ -305,4 +313,4 @@ pub fn is_input_expired(exp_date: Option) -> bool { } None => false, } -} \ No newline at end of file +} diff --git a/backend/src/model/config/mod.rs b/backend/src/model/config/mod.rs index 99a5094a5..d9eda0d3d 100644 --- a/backend/src/model/config/mod.rs +++ b/backend/src/model/config/mod.rs @@ -30,6 +30,7 @@ pub mod macros; mod favourites; mod geoip; mod library; +mod panel_api; pub use api::*; pub use api_proxy::*; @@ -60,4 +61,5 @@ pub use video_download::*; pub use web_auth::*; pub use web_ui::*; pub use favourites::*; -pub use library::*; \ No newline at end of file +pub use library::*; +pub use panel_api::*; \ No newline at end of file diff --git a/backend/src/model/config/panel_api.rs b/backend/src/model/config/panel_api.rs new file mode 100644 index 000000000..feb8ce6b1 --- /dev/null +++ b/backend/src/model/config/panel_api.rs @@ -0,0 +1,195 @@ +use crate::model::macros; +use shared::create_tuliprox_error_result; +use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::model::{PanelApiConfigDto, PanelApiQueryParamDto, PanelApiQueryParametersDto}; + +#[derive(Debug, Clone)] +pub struct PanelApiQueryParam { + pub key: String, + pub value: String, +} + +macros::from_impl!(PanelApiQueryParam); +impl From<&PanelApiQueryParamDto> for PanelApiQueryParam { + fn from(dto: &PanelApiQueryParamDto) -> Self { + Self { + key: dto.key.clone(), + value: dto.value.clone(), + } + } +} + +impl From<&PanelApiQueryParam> for PanelApiQueryParamDto { + fn from(instance: &PanelApiQueryParam) -> Self { + Self { + key: instance.key.clone(), + value: instance.value.clone(), + } + } +} + +#[derive(Debug, Clone)] +pub struct PanelApiQueryParameters { + pub client_info: Vec, + pub client_new: Vec, + pub client_renew: Vec, +} + +macros::from_impl!(PanelApiQueryParameters); +impl From<&PanelApiQueryParametersDto> for PanelApiQueryParameters { + fn from(dto: &PanelApiQueryParametersDto) -> Self { + Self { + client_info: dto.client_info.iter().map(PanelApiQueryParam::from).collect(), + client_new: dto.client_new.iter().map(PanelApiQueryParam::from).collect(), + client_renew: dto.client_renew.iter().map(PanelApiQueryParam::from).collect(), + } + } +} + +impl From<&PanelApiQueryParameters> for PanelApiQueryParametersDto { + fn from(instance: &PanelApiQueryParameters) -> Self { + Self { + client_info: instance.client_info.iter().map(PanelApiQueryParamDto::from).collect(), + client_new: instance.client_new.iter().map(PanelApiQueryParamDto::from).collect(), + client_renew: instance.client_renew.iter().map(PanelApiQueryParamDto::from).collect(), + } + } +} + + +impl PanelApiQueryParameters { + + fn validate_type_is_m3u(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + let typ = params + .iter() + .find(|p| p.key.trim().eq_ignore_ascii_case("type")) + .map(|p| p.value.trim().to_string()); + match typ { + Some(v) if v.eq_ignore_ascii_case("m3u") => Ok(()), + Some(v) => create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: unsupported type={v}, only m3u is supported"), + None => create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: missing required query param 'type=m3u'"), + } + } + + fn require_api_key_param(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { + let api_key = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("api_key")); + let Some(api_key) = api_key else { + return create_tuliprox_error_result!( + TuliproxErrorKind::Info, + "panel_api: {section} must contain query param 'api_key' (use value 'auto')" + ); + }; + if api_key.value.trim().is_empty() { + return create_tuliprox_error_result!( + TuliproxErrorKind::Info, + "panel_api: {section} query param 'api_key' must not be empty (use value 'auto')" + ); + } + Ok(()) + } + + fn require_username_password_params_auto(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { + let username = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("username")); + let password = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("password")); + if username.is_none() || password.is_none() { + return create_tuliprox_error_result!( + TuliproxErrorKind::Info, + "panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')" + ); + } + if !username.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) + || !password.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) + { + return create_tuliprox_error_result!( + TuliproxErrorKind::Info, + "panel_api: {section} requires 'username: auto' and 'password: auto' (credentials must not be hardcoded)" + ); + } + Ok(()) + } + + fn validate_client_info_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + Self::require_api_key_param(params, "query_parameter.client_info")?; + Self::require_username_password_params_auto(params, "query_parameter.client_info")?; + Ok(()) + } + + fn validate_client_new_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + Self::require_api_key_param(params, "query_parameter.client_new")?; + Self::validate_type_is_m3u(params)?; + if params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("user")) { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new must not contain query param 'user'"); + } + Ok(()) + } + + fn validate_client_renew_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + Self::require_api_key_param(params, "query_parameter.client_renew")?; + Self::validate_type_is_m3u(params)?; + Self::require_username_password_params_auto(params, "query_parameter.client_renew")?; + Ok(()) + } + + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + Self::validate_client_info_params(&self.client_info)?; + Self::validate_client_new_params(&self.client_new)?; + Self::validate_client_renew_params(&self.client_renew)?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct PanelApiConfig { + pub enabled: bool, + pub url: String, + pub api_key: Option, + pub query_parameter: PanelApiQueryParameters, +} + +macros::from_impl!(PanelApiConfig); +impl From<&PanelApiConfigDto> for PanelApiConfig { + fn from(dto: &PanelApiConfigDto) -> Self { + Self { + enabled: dto.enabled, + url: dto.url.clone(), + api_key: dto.api_key.clone(), + query_parameter: PanelApiQueryParameters::from(&dto.query_parameter), + } + } +} + +impl From<&PanelApiConfig> for PanelApiConfigDto { + fn from(instance: &PanelApiConfig) -> Self { + Self { + enabled: instance.enabled, + url: instance.url.clone(), + api_key: instance.api_key.clone(), + query_parameter: PanelApiQueryParametersDto::from(&instance.query_parameter), + + } + } +} + +impl PanelApiConfig { + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + if !self.enabled { + return Ok(()); + } + if self.url.trim().is_empty() { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: url is missing"); + } + if self.api_key.as_ref().is_none_or(|k| k.trim().is_empty()) { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: api_key is missing"); + } + if self.query_parameter.client_info.is_empty() + || self.query_parameter.client_new.is_empty() + || self.query_parameter.client_renew.is_empty() + { + return create_tuliprox_error_result!( + TuliproxErrorKind::Info, + "panel_api: query_parameter.client_info/client_new/client_renew must be configured" + ); + } + self.query_parameter.prepare() + } +} \ No newline at end of file diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 63463bc42..6c2c72078 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -62,9 +62,10 @@ pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) } pub fn apply_favourites_to_playlist( - _playlist: &mut Vec, - _favourites_cfg: Option<&Vec>, + _playlist: &mut [PlaylistGroup], + _favourites_cfg: Option<&[ConfigFavourites]>, ) { + // TODO implement favourites // if let Some(favourites) = favourites_cfg { // let mut fav_groups: HashMap> = HashMap::new(); // @@ -101,7 +102,7 @@ pub fn apply_favourites_to_playlist( fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { if let Some(mut filtered_playlist) = apply_filter_to_playlist(playlist, &target.filter) { - apply_favourites_to_playlist(&mut filtered_playlist, target.favourites.as_ref()); + apply_favourites_to_playlist(&mut filtered_playlist, target.favourites.as_deref()); Some(filtered_playlist) } else { None diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index 8ba0faffa..088664fc7 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -302,7 +302,7 @@ pub async fn read_api_proxy(config: &AppConfig, resolve_env: bool) -> Option(file_path: &str, backup_dir: &str, config: &T, default_name: &str) -> Result<(), TuliproxError> +async fn write_config_file(file_path: &str, backup_dir: &str, config: &T, default_name: &str, formatter: Option<&(dyn Fn(&str) -> String + Send + Sync)>) -> Result<(), TuliproxError> where T: ?Sized + Serialize, { @@ -317,20 +317,213 @@ where } info!("Saving file to {}", &path.to_str().unwrap_or("?")); - let serialized = serde_yaml::to_string(config) + let mut serialized = serde_yaml::to_string(config) .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Info, "Could not serialize file {}: {}", &path.to_str().unwrap_or("?"), err))?; + if let Some(format) = formatter { + serialized = format(&serialized); + } fs::write(&path, serialized) .await .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Info, "Could not write file {}: {}", &path.to_str().unwrap_or("?"), err)) } +fn format_sources_yaml_panel_api_query_params_flow_style_impl(yaml: &str) -> String { + let has_trailing_newline = yaml.ends_with('\n'); + let lines: Vec<&str> = yaml.split_terminator('\n').collect(); + let mut out: Vec = Vec::with_capacity(lines.len()); + + let mut panel_api_indent: Option = None; + let mut query_indent: Option = None; + let mut section_indent: Option = None; + + let mut i = 0usize; + while i < lines.len() { + let line = lines[i]; + let trimmed = line.trim_start(); + let indent = line.len().saturating_sub(trimmed.len()); + + if !trimmed.is_empty() && !trimmed.starts_with('#') { + if let Some(pi) = panel_api_indent { + if indent <= pi && trimmed != "panel_api:" { + panel_api_indent = None; + query_indent = None; + section_indent = None; + } + } + if let Some(qi) = query_indent { + if indent <= qi && trimmed != "query_parameter:" { + query_indent = None; + section_indent = None; + } + } + if let Some(si) = section_indent { + if indent < si { + section_indent = None; + } + } + + if panel_api_indent.is_none() && trimmed == "panel_api:" { + panel_api_indent = Some(indent); + } else if panel_api_indent.is_some() && query_indent.is_none() && trimmed == "query_parameter:" { + query_indent = Some(indent); + } else if query_indent.is_some() + && matches!(trimmed, "client_info:" | "client_new:" | "client_renew:") + { + section_indent = Some(indent); + } + } + + let in_target_section = panel_api_indent.is_some() && query_indent.is_some() && section_indent.is_some(); + if in_target_section + && indent == section_indent.unwrap_or(usize::MAX) + && trimmed.starts_with("- key:") + && !trimmed.starts_with("- {") + && i + 1 < lines.len() + { + let next = lines[i + 1]; + let next_trimmed = next.trim_start(); + let next_indent = next.len().saturating_sub(next_trimmed.len()); + + if next_trimmed.starts_with("value:") + && next_indent == indent.saturating_add(2) + { + let key_val = trimmed + .strip_prefix("- key:") + .unwrap_or_default() + .trim(); + let value_val = next_trimmed + .strip_prefix("value:") + .unwrap_or_default() + .trim(); + + let new_line = format!( + "{}- {{ key: {}, value: {} }}", + " ".repeat(indent), + key_val, + value_val + ); + out.push(new_line); + i += 2; + continue; + } + } + + out.push(line.to_string()); + i += 1; + } + + if has_trailing_newline { + out.join("\n") + "\n" + } else { + out.join("\n") + } +} + +fn parse_yaml_key_value_line(s: &str) -> Option<(String, String)> { + let (key, value) = s.split_once(':')?; + let key = key.trim(); + if key.is_empty() { + return None; + } + let value = value.trim(); + let value = if value.is_empty() { "null" } else { value }; + Some((key.to_string(), value.to_string())) +} + +fn format_sources_yaml_aliases_flow_style_impl(yaml: &str) -> String { + let has_trailing_newline = yaml.ends_with('\n'); + let lines: Vec<&str> = yaml.split_terminator('\n').collect(); + let mut out: Vec = Vec::with_capacity(lines.len()); + + let mut aliases_indent: Option = None; + + let mut i = 0usize; + while i < lines.len() { + let line = lines[i]; + let trimmed = line.trim_start(); + let indent = line.len().saturating_sub(trimmed.len()); + + if !trimmed.is_empty() && !trimmed.starts_with('#') { + if let Some(ai) = aliases_indent { + if indent < ai || (indent == ai && trimmed != "aliases:" && !trimmed.starts_with("- ")) { + aliases_indent = None; + } + } + if trimmed == "aliases:" { + aliases_indent = Some(indent); + } + } + + // Convert: + // aliases: + // - name: foo + // url: http://... + // Into: + // - { name: foo, url: http://... } + if aliases_indent.is_some() && trimmed.starts_with("- ") && !trimmed.starts_with("- {") { + let item_indent = indent; + let first = trimmed.strip_prefix("- ").unwrap_or(trimmed); + if let Some((k, v)) = parse_yaml_key_value_line(first) { + let mut parts: Vec<(String, String)> = vec![(k, v)]; + let mut j = i + 1; + while j < lines.len() { + let l2 = lines[j]; + let t2 = l2.trim_start(); + let ind2 = l2.len().saturating_sub(t2.len()); + if t2.is_empty() || t2.starts_with('#') { + break; + } + if ind2 == item_indent.saturating_add(2) && !t2.starts_with("- ") { + if let Some((k2, v2)) = parse_yaml_key_value_line(t2) { + parts.push((k2, v2)); + j += 1; + continue; + } + } + break; + } + + let formatted = parts + .iter() + .map(|(kk, vv)| format!("{kk}: {vv}")) + .collect::>() + .join(", "); + out.push(format!("{}- {{ {} }}", " ".repeat(item_indent), formatted)); + i = j; + continue; + } + } + + out.push(line.to_string()); + i += 1; + } + + if has_trailing_newline { + out.join("\n") + "\n" + } else { + out.join("\n") + } +} + +pub fn format_sources_yaml_panel_api_query_params_flow_style(yaml: &str) -> String { + let yaml = format_sources_yaml_panel_api_query_params_flow_style_impl(yaml); + format_sources_yaml_aliases_flow_style_impl(&yaml) +} + pub async fn save_api_proxy(file_path: &str, backup_dir: &str, config: &ApiProxyConfigDto) -> Result<(), TuliproxError> { - write_config_file(file_path, backup_dir, config, "api-proxy.yml").await + write_config_file(file_path, backup_dir, config, "api-proxy.yml", None).await } pub async fn save_main_config(file_path: &str, backup_dir: &str, config: &ConfigDto) -> Result<(), TuliproxError> { - write_config_file(file_path, backup_dir, config, "config.yml").await + write_config_file(file_path, backup_dir, config, "config.yml", None).await +} + +pub async fn save_sources_config(file_path: &str, backup_dir: &str, config: &T) -> Result<(), TuliproxError> +where + T: ?Sized + Serialize +{ + write_config_file(file_path, backup_dir, config, "source.yml", Some(&format_sources_yaml_panel_api_query_params_flow_style)).await } pub fn resolve_env_var(value: &str) -> String { @@ -349,10 +542,56 @@ pub fn resolve_env_var(value: &str) -> String { #[cfg(test)] mod tests { use crate::utils::resolve_env_var; + use crate::utils::format_sources_yaml_panel_api_query_params_flow_style; #[test] fn test_resolve() { let resolved = resolve_env_var("${env:HOME}"); assert_eq!(resolved, std::env::var("HOME").unwrap()); } + + #[test] + fn test_sources_yaml_panel_api_query_params_flow_style() { + let input = r"sources: +- inputs: + - name: demo + panel_api: + url: https://example.invalid/api.php + query_parameter: + client_info: + - key: api_key + value: auto + - key: sub + value: '1' + client_new: + - key: type + value: m3u +"; + let out = format_sources_yaml_panel_api_query_params_flow_style(input); + assert!(out.contains("- { key: api_key, value: auto }")); + assert!(out.contains("- { key: sub, value: '1' }")); + assert!(out.contains("- { key: type, value: m3u }")); + assert!(!out.contains("- key: api_key\n")); + } + + #[test] + fn test_sources_yaml_aliases_flow_style() { + let input = r"sources: +- inputs: + - name: demo + aliases: + - name: demo-u1 + url: http://line.example.invalid + username: u1 + password: p1 + priority: 0 + max_connections: 1 + exp_date: 123 +"; + let out = format_sources_yaml_panel_api_query_params_flow_style(input); + assert!( + out.contains("- { name: demo-u1, url: http://line.example.invalid, username: u1, password: p1, priority: 0, max_connections: 1, exp_date: 123 }"), + "out:\n{out}" + ); + } } diff --git a/backend/src/utils/geoip.rs b/backend/src/utils/geoip.rs index 07af9ddf8..5e692a978 100644 --- a/backend/src/utils/geoip.rs +++ b/backend/src/utils/geoip.rs @@ -114,7 +114,7 @@ mod test { if let Some(cc) = geo_ip.lookup("72.13.24.23") { assert_eq!(cc, "US"); } else { - assert!(false); + panic!("GeoIP lookup returned no result"); } } -} \ No newline at end of file +} diff --git a/config/config.yml b/config/config.yml index dbea13148..00bee301a 100644 --- a/config/config.yml +++ b/config/config.yml @@ -74,7 +74,7 @@ reverse_proxy: shared_burst_buffer_mb: 12 # default 12 MB library: - enabled: false + enabled: true scan_directories: - path: "/media/movies" enabled: true @@ -98,20 +98,19 @@ library: # Metadata configuration metadata: # Location to store metadata cache (JSON files) - storage_location: "${env:TULIPROX_HOME}/library_metadata" + path: "${env:TULIPROX_HOME}/library_metadata" # TMDB (The Movie Database) API integration tmdb: - enabled: false - api_key: "" # Get your API key from https://www.themoviedb.org/settings/api + enabled: true + # api_key: "" # Get your API key from https://www.themoviedb.org/settings/api rate_limit_ms: 250 # Milliseconds between API calls (default: 250ms) # Fallback to filename parsing if no metadata found - fallback_to_filename_parsing: true + fallback_to_filename: true # Metadata output formats formats: - - "json" # Always stores JSON metadata # - "nfo" # Optionally write Kodi-compatible NFO files # Classification rules for movies vs series @@ -127,7 +126,7 @@ library: # Playlist integration playlist: # Group name for movies in M3U/Xtream playlist - movie_group_name: "Local Movies" + movie_category: "Local Movies" # Group name for series in M3U/Xtream playlist - series_group_name: "Local Series" + series_category: "Local Series" diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index c7c81128c..2fea0d3cd 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -1,10 +1,11 @@ [package] name = "frontend" -version = "3.2.21" +version = "3.2.22" edition = "2021" +rust-version = "1.87.0" [dependencies] -shared = { version = "3.2.21", path = "../shared" } +shared = { version = "3.2.22", path = "../shared" } chrono = "0" yew = "0.21" yew-router = "0.18" diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index 54f1ca7c2..8c1730e08 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -346,6 +346,15 @@ "API_CONFIGURATION": "API Configuration", "HEADER_NAME": "Header Name", "HEADER_VALUE": "Header Value", + "API_KEY": "API-Key", + "PANEL": "Panel", + "PANEL_CLIENT_INFO": "Client info", + "PANEL_CLIENT_NEW": "Client new", + "PANEL_CLIENT_RENEW": "Client renew", + "PANEL_STATUS_READY": "Ready", + "PANEL_STATUS_INVALID": "Invalid", + "PANEL_STATUS_DISABLED": "Disabled", + "VALIDATION": "Validation", "TEMPLATES": "Templates" }, "TITLE": { @@ -386,10 +395,15 @@ "MESSAGE": "Enter Your Credentials" }, "SAVE": { + "NO_CHANGES": "No changes", "MAIN_CONFIG": { "SUCCESS": "Main config saved!", "FAIL": "Failed to save the main config!" }, + "SOURCES_CONFIG": { + "SUCCESS": "Sources config saved!", + "FAIL": "Failed to save the sources config!" + }, "BOUQUET": { "SUCCESS": "Successfully saved bouquet", "FAIL": "Failed to save the bouquet!" @@ -452,6 +466,10 @@ }, "WEB_UI": { "PATH": "Configuration of a path prefix, allowing it to be accessed behind a reverse proxy under a custom subpath. like https://yourdomain.com/tuliprox" + }, + "PANEL": { + "INFO": "Configure provisioning via a generic provider panel API per input. Use query param value 'auto' for api_key and credentials where required.", + "ENABLE": "Enable panel API for this input to configure provisioning." } } }, diff --git a/frontend/scss/app/_component.scss b/frontend/scss/app/_component.scss index bf2dc35d8..e21003fcc 100644 --- a/frontend/scss/app/_component.scss +++ b/frontend/scss/app/_component.scss @@ -66,6 +66,7 @@ @forward "components/accordion"; @forward "components/cvs_table"; @forward "components/config/config_view"; +@forward "components/config/panel_config_view"; @forward "components/tabset"; @forward "components/select"; @forward "components/form"; diff --git a/frontend/scss/app/components/config/_config_view.scss b/frontend/scss/app/components/config/_config_view.scss index 1b54ae5ba..ec397a3a1 100644 --- a/frontend/scss/app/components/config/_config_view.scss +++ b/frontend/scss/app/components/config/_config_view.scss @@ -266,4 +266,5 @@ box-sizing: border-box; align-items: center; } -} \ No newline at end of file +} + diff --git a/frontend/scss/app/components/config/_panel_config_view.scss b/frontend/scss/app/components/config/_panel_config_view.scss new file mode 100644 index 000000000..d0eea0df4 --- /dev/null +++ b/frontend/scss/app/components/config/_panel_config_view.scss @@ -0,0 +1,153 @@ +.tp__panel-config-view { + &__info-text { + color: var(--modest-text-color); + line-height: 1.35; + } + + &__input-header { + display: flex; + flex-flow: row nowrap; + justify-content: space-between; + align-items: flex-start; + gap: var(--gap-default); + box-sizing: border-box; + width: 100%; + } + + &__input-title { + display: flex; + flex-flow: column; + gap: var(--gap-micro); + } + + &__input-subtitle { + color: var(--modest-text-color); + font-size: 0.9rem; + } + + &__input-status { + display: flex; + flex-flow: column; + align-items: flex-end; + gap: var(--gap-small); + } + + &__toggle { + display: flex; + flex-flow: row nowrap; + align-items: center; + gap: var(--gap-small); + } + + &__disabled-hint { + color: var(--modest-text-color); + border: 1px dashed var(--card-border-color); + border-radius: var(--border-radius); + padding: var(--padding-small); + } + + &__summary { + display: grid; + grid-template-columns: 1fr 1fr; + gap: var(--gap-default); + width: 100%; + + @media (max-width: size.$mobile-breakpoint) { + grid-template-columns: 1fr; + } + } + + &__section { + display: flex; + flex-flow: column; + gap: var(--gap-default); + padding-top: var(--padding-default); + border-top: 1px solid var(--card-border-color); + } + + &__section-header { + display: flex; + flex-flow: row nowrap; + justify-content: space-between; + align-items: center; + gap: var(--gap-default); + + h2 { + font-size: 1.05rem; + margin: 0; + } + } + + &__section-actions { + display: flex; + flex-flow: row nowrap; + gap: var(--gap-small); + } + + &__params { + display: flex; + flex-flow: column; + gap: var(--gap-small); + } + + &__param-row { + display: grid; + grid-template-columns: 1fr 1fr auto; + gap: var(--gap-small); + align-items: flex-end; + + .tp__form-field { + min-width: 0; + } + } + + &__param-row--view { + grid-template-columns: 1fr; + align-items: stretch; + } + + &__param-remove { + margin-bottom: 0.35rem; + } + + &__param-view { + display: flex; + flex-flow: row nowrap; + justify-content: space-between; + align-items: center; + gap: var(--gap-default); + border: 1px solid var(--card-border-color); + border-radius: var(--border-radius); + padding: var(--padding-small); + width: 100%; + + .k { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + color: var(--label-color); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 40%; + } + + .v { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + color: var(--modest-text-color); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 60%; + } + } + + &__errors { + border-top: 1px solid var(--card-border-color); + padding-top: var(--padding-default); + color: var(--warn-color); + + ul { + padding-left: 1.25rem; + margin: 0; + } + } +} diff --git a/frontend/src/app/components/api_user/playlist.rs b/frontend/src/app/components/api_user/playlist.rs index 118550665..b877a2366 100644 --- a/frontend/src/app/components/api_user/playlist.rs +++ b/frontend/src/app/components/api_user/playlist.rs @@ -68,7 +68,6 @@ fn to_playlist_cluster(count: (usize, usize, usize), bouquet: Option<&Rc Html { let translate = use_translation(); diff --git a/frontend/src/app/components/config/config_page.rs b/frontend/src/app/components/config/config_page.rs index 023f8d5dc..09d7f4bac 100644 --- a/frontend/src/app/components/config/config_page.rs +++ b/frontend/src/app/components/config/config_page.rs @@ -1,6 +1,6 @@ use shared::error::TuliproxError; use shared::info_err; -use shared::model::{ConfigApiDto, HdHomeRunConfigDto, IpCheckConfigDto, LogConfigDto, MainConfigDto, MessagingConfigDto, ProxyConfigDto, ReverseProxyConfigDto, SchedulesConfigDto, VideoConfigDto, WebUiConfigDto}; +use shared::model::{ConfigApiDto, HdHomeRunConfigDto, IpCheckConfigDto, LogConfigDto, MainConfigDto, MessagingConfigDto, ProxyConfigDto, ReverseProxyConfigDto, SchedulesConfigDto, SourcesConfigDto, VideoConfigDto, WebUiConfigDto}; use std::fmt; use std::str::FromStr; @@ -15,6 +15,7 @@ pub const LABEL_HDHOMERUN_CONFIG: &str = "LABEL.HDHOMERUN_CONFIG"; pub const LABEL_PROXY_CONFIG: &str = "LABEL.PROXY"; pub const LABEL_IP_CHECK_CONFIG: &str = "LABEL.IP_CHECK"; pub const LABEL_VIDEO_CONFIG: &str = "LABEL.VIDEO"; +pub const LABEL_PANEL_CONFIG: &str = "LABEL.PANEL"; const MAIN_PAGE: &str = "main"; const API_PAGE: &str = "api"; @@ -27,6 +28,7 @@ const HDHOMERUN_PAGE: &str = "hdhomerun"; const PROXY_PAGE: &str = "proxy"; const IPCHECK_PAGE: &str = "ipcheck"; const VIDEO_PAGE: &str = "video"; +const PANEL_PAGE: &str = "panel"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] pub enum ConfigPage { @@ -41,6 +43,7 @@ pub enum ConfigPage { HdHomerun, Proxy, IpCheck, + Panel, } impl FromStr for ConfigPage { @@ -59,6 +62,7 @@ impl FromStr for ConfigPage { HDHOMERUN_PAGE => Ok(ConfigPage::HdHomerun), PROXY_PAGE => Ok(ConfigPage::Proxy), IPCHECK_PAGE => Ok(ConfigPage::IpCheck), + PANEL_PAGE => Ok(ConfigPage::Panel), _ => Err(info_err!(format!("Unknown config page: {s}"))), } } @@ -78,6 +82,7 @@ impl fmt::Display for ConfigPage { ConfigPage::HdHomerun => HDHOMERUN_PAGE, ConfigPage::Proxy => PROXY_PAGE, ConfigPage::IpCheck => IPCHECK_PAGE, + ConfigPage::Panel => PANEL_PAGE, }; write!(f, "{s}") } @@ -96,6 +101,7 @@ pub enum ConfigForm { HdHomerun(bool, HdHomeRunConfigDto), Proxy(bool, ProxyConfigDto), IpCheck(bool, IpCheckConfigDto), + Panel(bool, SourcesConfigDto), } impl ConfigForm { @@ -110,6 +116,7 @@ impl ConfigForm { | ConfigForm::ReverseProxy(true, _) | ConfigForm::HdHomerun(true, _) | ConfigForm::Proxy(true, _) - | ConfigForm::IpCheck(true, _)) + | ConfigForm::IpCheck(true, _) + | ConfigForm::Panel(true, _)) } } diff --git a/frontend/src/app/components/config/config_update.rs b/frontend/src/app/components/config/config_update.rs index 23f63a5c3..73aff9f90 100644 --- a/frontend/src/app/components/config/config_update.rs +++ b/frontend/src/app/components/config/config_update.rs @@ -32,6 +32,7 @@ pub fn update_config(config: &mut ConfigDto, forms: Vec) { ConfigForm::HdHomerun(_, mut hdhr_cfg) => set_config_field!(config, hdhr_cfg, hdhomerun), ConfigForm::Proxy(_, mut proxy_cfg) => set_config_field!(config, proxy_cfg, proxy), ConfigForm::IpCheck(_, mut ipcheck_cfg) => set_config_field!(config, ipcheck_cfg, ipcheck), + ConfigForm::Panel(_, _) => {} } } -} \ No newline at end of file +} diff --git a/frontend/src/app/components/config/config_view.rs b/frontend/src/app/components/config/config_view.rs index 97b6f4280..55443374b 100644 --- a/frontend/src/app/components/config/config_view.rs +++ b/frontend/src/app/components/config/config_view.rs @@ -1,13 +1,14 @@ -use crate::app::components::config::config_page::{ConfigForm, ConfigPage, LABEL_API_CONFIG, LABEL_HDHOMERUN_CONFIG, LABEL_IP_CHECK_CONFIG, LABEL_LOG_CONFIG, LABEL_MAIN_CONFIG, LABEL_MESSAGING_CONFIG, LABEL_PROXY_CONFIG, LABEL_REVERSE_PROXY_CONFIG, LABEL_SCHEDULES_CONFIG, LABEL_VIDEO_CONFIG, LABEL_WEB_UI_CONFIG}; +use crate::app::components::config::config_page::{ConfigForm, ConfigPage, LABEL_API_CONFIG, LABEL_HDHOMERUN_CONFIG, LABEL_IP_CHECK_CONFIG, LABEL_LOG_CONFIG, LABEL_MAIN_CONFIG, LABEL_MESSAGING_CONFIG, LABEL_PANEL_CONFIG, LABEL_PROXY_CONFIG, LABEL_REVERSE_PROXY_CONFIG, LABEL_SCHEDULES_CONFIG, LABEL_VIDEO_CONFIG, LABEL_WEB_UI_CONFIG}; use crate::app::components::config::config_view_context::ConfigViewContext; -use crate::app::components::config::{ApiConfigView, HdHomerunConfigView, IpCheckConfigView, LogConfigView, MainConfigView, MessagingConfigView, ProxyConfigView, ReverseProxyConfigView, SchedulesConfigView, VideoConfigView, WebUiConfigView}; +use crate::app::components::config::{ApiConfigView, HdHomerunConfigView, IpCheckConfigView, LogConfigView, MainConfigView, MessagingConfigView, PanelConfigView, ProxyConfigView, ReverseProxyConfigView, SchedulesConfigView, VideoConfigView, WebUiConfigView}; use crate::app::components::{Card, TabItem, TabSet, TextButton}; use crate::html_if; use std::str::FromStr; +use log::warn; use yew::platform::spawn_local; use yew::prelude::*; use yew_i18n::use_translation; -use shared::model::ConfigDto; +use shared::model::{ConfigDto, SourcesConfigDto}; use crate::app::components::config::config_update::update_config; use crate::app::{ConfigContext}; use crate::hooks::use_service_context; @@ -46,7 +47,8 @@ fn config_form_to_config_page(form: &ConfigForm) -> ConfigPage { ConfigForm::ReverseProxy(_, _) => ConfigPage::ReverseProxy, ConfigForm::HdHomerun(_, _) => ConfigPage::HdHomerun, ConfigForm::Proxy(_, _) => ConfigPage::Proxy, - ConfigForm::IpCheck(_, _) => ConfigPage::IpCheck + ConfigForm::IpCheck(_, _) => ConfigPage::IpCheck, + ConfigForm::Panel(_, _) => ConfigPage::Panel, } } @@ -63,6 +65,7 @@ struct ConfigFormState { pub hd_homerun: Option, pub proxy: Option, pub ipcheck: Option, + pub panel: Option, } #[function_component] @@ -93,7 +96,7 @@ pub fn ConfigView() -> Html { let forms: &ConfigFormState = forms; let modified_pages = collect_modified!(forms, [ main, api, log, schedules, video, messaging, web_ui, - reverse_proxy, hd_homerun, proxy, ipcheck + reverse_proxy, hd_homerun, proxy, ipcheck, panel ]).iter() .map(config_form_to_config_page) .collect::>(); @@ -109,6 +112,7 @@ pub fn ConfigView() -> Html { (ConfigPage::HdHomerun, LABEL_HDHOMERUN_CONFIG, html! { }, "HdHomerunConfig"), (ConfigPage::Proxy, LABEL_PROXY_CONFIG, html! { }, "ProxyConfig"), (ConfigPage::IpCheck, LABEL_IP_CHECK_CONFIG, html! { }, "IpCheckConfig"), + (ConfigPage::Panel, LABEL_PANEL_CONFIG, html! { }, "Settings"), (ConfigPage::Video, LABEL_VIDEO_CONFIG, html! { }, "VideoConfig"), ]; @@ -145,39 +149,85 @@ pub fn ConfigView() -> Html { let forms = &*get_form_state; let modified_forms: Vec = collect_modified!(forms, [ main, api, log, schedules, video, messaging, web_ui, - reverse_proxy, hd_homerun, proxy, ipcheck + reverse_proxy, hd_homerun, proxy, ipcheck, panel ]); - if !modified_forms.is_empty() { - let mut config_dto = config_ctx.config.as_ref().map_or_else(ConfigDto::default, - |app_cfg| app_cfg.config.clone()); - update_config(&mut config_dto, modified_forms); - match config_dto.prepare(false) { - Ok(_) => { - let services = services.clone(); - let translate = translate.clone(); - let set_edit_mode = set_edit_mode.clone(); - spawn_local(async move { - match services.config.save_config(config_dto).await{ - Ok(()) => { - services.toastr.success(translate.t("MESSAGES.SAVE.MAIN_CONFIG.SUCCESS")); - set_edit_mode.set(false); - let _cfg = services.config.get_server_config().await; - }, - Err(err) => { - services.toastr.error(translate.t("MESSAGES.SAVE.MAIN_CONFIG.FAIL")); - services.toastr.error(err.to_string()); - } - } - }); - } - Err(err) => { - services.toastr.error(err.to_string()); + if modified_forms.is_empty() { + set_edit_mode.set(false); + services.toastr.info(translate.t("MESSAGES.SAVE.NO_CHANGES")); + return; + } + + let mut modified_main_forms = Vec::new(); + let mut modified_sources: Option = None; + for form in modified_forms { + match form { + ConfigForm::Panel(_, sources) => modified_sources = Some(sources), + other => modified_main_forms.push(other), + } + } + + let mut modified_main_dto: Option = None; + if !modified_main_forms.is_empty() { + let mut config_dto = config_ctx + .config + .as_ref() + .map_or_else(ConfigDto::default, |app_cfg| app_cfg.config.clone()); + update_config(&mut config_dto, modified_main_forms); + if let Err(err) = config_dto.prepare(false) { + services.toastr.error(err.to_string()); + return; + } + modified_main_dto = Some(config_dto); + } + + if let Some(sources) = modified_sources.as_mut() { + if let Err(err) = sources.prepare(false, None) { + services.toastr.error(err.to_string()); + return; + } + } + + let services = services.clone(); + let translate = translate.clone(); + let set_edit_mode = set_edit_mode.clone(); + spawn_local(async move { + let mut ok = true; + + if let Some(config_dto) = modified_main_dto { + match services.config.save_config(config_dto).await { + Ok(()) => { + services.toastr.success(translate.t("MESSAGES.SAVE.MAIN_CONFIG.SUCCESS")); + } + Err(err) => { + ok = false; + services.toastr.error(translate.t("MESSAGES.SAVE.MAIN_CONFIG.FAIL")); + services.toastr.error(err.to_string()); + } } } - } else { - set_edit_mode.set(false); - } + + if let Some(sources_dto) = modified_sources { + match services.config.save_sources(sources_dto).await { + Ok(()) => { + services.toastr.success(translate.t("MESSAGES.SAVE.SOURCES_CONFIG.SUCCESS")); + } + Err(err) => { + ok = false; + services.toastr.error(translate.t("MESSAGES.SAVE.SOURCES_CONFIG.FAIL")); + services.toastr.error(err.to_string()); + } + } + } + + if ok { + set_edit_mode.set(false); + if services.config.get_server_config().await.is_none() { + // Log but don't fail - save succeeded, refresh is best-effort + warn!("Config refresh failed"); + } + } + }); }) }; @@ -198,6 +248,7 @@ pub fn ConfigView() -> Html { ConfigForm::HdHomerun(_, _) => new_state.hd_homerun = Some(form_data), ConfigForm::Proxy(_, _) => new_state.proxy = Some(form_data), ConfigForm::IpCheck(_, _) => new_state.ipcheck = Some(form_data), + ConfigForm::Panel(_, _) => new_state.panel = Some(form_data), }; set_form_state.set(new_state); }) diff --git a/frontend/src/app/components/config/mod.rs b/frontend/src/app/components/config/mod.rs index 0c5778e08..2a3cf3791 100644 --- a/frontend/src/app/components/config/mod.rs +++ b/frontend/src/app/components/config/mod.rs @@ -16,6 +16,7 @@ mod config_page; mod hdhomerun_device_view; mod config_update; mod log_config_view; +mod panel_config_view; pub use config_view::*; pub use main_config_view::*; @@ -29,4 +30,5 @@ pub use proxy_config_view::*; pub use ipcheck_config_view::*; pub use video_config_view::*; pub use log_config_view::*; -pub use macros::*; \ No newline at end of file +pub use panel_config_view::*; +pub use macros::*; diff --git a/frontend/src/app/components/config/panel_config_view.rs b/frontend/src/app/components/config/panel_config_view.rs new file mode 100644 index 000000000..8796d4dcb --- /dev/null +++ b/frontend/src/app/components/config/panel_config_view.rs @@ -0,0 +1,545 @@ +use crate::app::components::input::Input; +use crate::app::components::{Card, Chip, IconButton, ToggleSwitch}; +use crate::app::components::config::config_page::{ConfigForm, LABEL_PANEL_CONFIG}; +use crate::app::components::config::config_view_context::ConfigViewContext; +use crate::app::context::ConfigContext; +use crate::html_if; +use shared::model::{ConfigInputDto, PanelApiConfigDto, PanelApiQueryParamDto, SourcesConfigDto}; +use std::rc::Rc; +use yew::prelude::*; +use yew_i18n::use_translation; + +const LABEL_ENABLED: &str = "LABEL.ENABLED"; +const LABEL_URL: &str = "LABEL.URL"; +const LABEL_API_KEY: &str = "LABEL.API_KEY"; +const LABEL_STATUS_READY: &str = "LABEL.PANEL_STATUS_READY"; +const LABEL_STATUS_INVALID: &str = "LABEL.PANEL_STATUS_INVALID"; +const LABEL_STATUS_DISABLED: &str = "LABEL.PANEL_STATUS_DISABLED"; +const LABEL_VALIDATION: &str = "LABEL.VALIDATION"; +const HINT_PANEL_INFO: &str = "HINT.CONFIG.PANEL.INFO"; +const HINT_PANEL_ENABLE: &str = "HINT.CONFIG.PANEL.ENABLE"; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum PanelSection { + Info, + New, + Renew, +} + +impl PanelSection { + fn label_key(&self) -> &'static str { + match self { + Self::Info => "LABEL.PANEL_CLIENT_INFO", + Self::New => "LABEL.PANEL_CLIENT_NEW", + Self::Renew => "LABEL.PANEL_CLIENT_RENEW", + } + } +} + +#[derive(Debug, Clone, PartialEq)] +struct PanelConfigFormState { + form: SourcesConfigDto, + modified: bool, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Clone)] +enum PanelConfigFormAction { + SetAll(SourcesConfigDto), + SetEnabled { source_idx: usize, input_idx: usize, enabled: bool }, + SetPanelUrl { source_idx: usize, input_idx: usize, url: String }, + SetApiKey { source_idx: usize, input_idx: usize, api_key: String }, + AddParam { source_idx: usize, input_idx: usize, section: PanelSection }, + RemoveParam { source_idx: usize, input_idx: usize, section: PanelSection, param_idx: usize }, + SetParamKey { source_idx: usize, input_idx: usize, section: PanelSection, param_idx: usize, key: String }, + SetParamValue { source_idx: usize, input_idx: usize, section: PanelSection, param_idx: usize, value: String }, + EnsureRequired { source_idx: usize, input_idx: usize, section: PanelSection }, +} + +fn params_mut(panel: &mut PanelApiConfigDto, section: PanelSection) -> &mut Vec { + match section { + PanelSection::Info => &mut panel.query_parameter.client_info, + PanelSection::New => &mut panel.query_parameter.client_new, + PanelSection::Renew => &mut panel.query_parameter.client_renew, + } +} + +fn has_param(params: &[PanelApiQueryParamDto], key: &str) -> bool { + params.iter().any(|p| p.key.trim().eq_ignore_ascii_case(key)) +} + +fn get_param_value(params: &[PanelApiQueryParamDto], key: &str) -> Option { + params + .iter() + .find(|p| p.key.trim().eq_ignore_ascii_case(key)) + .map(|p| p.value.trim().to_string()) +} + +fn validate_panel(panel: Option<&PanelApiConfigDto>) -> Vec { + let mut errors = vec![]; + let Some(panel) = panel else { + return errors; + }; + + if panel.url.trim().is_empty() { + errors.push("Missing panel url".to_string()); + } + if panel.api_key.as_ref().is_none_or(|k| k.trim().is_empty()) { + errors.push("Missing api_key".to_string()); + } + + let sections = [ + (PanelSection::Info, &panel.query_parameter.client_info), + (PanelSection::New, &panel.query_parameter.client_new), + (PanelSection::Renew, &panel.query_parameter.client_renew), + ]; + for (section, params) in sections { + if params.is_empty() { + errors.push(match section { + PanelSection::Info => "client_info: empty".to_string(), + PanelSection::New => "client_new: empty".to_string(), + PanelSection::Renew => "client_renew: empty".to_string(), + }); + continue; + } + if !has_param(params, "api_key") { + errors.push(match section { + PanelSection::Info => "client_info: missing api_key param".to_string(), + PanelSection::New => "client_new: missing api_key param".to_string(), + PanelSection::Renew => "client_renew: missing api_key param".to_string(), + }); + } + match section { + PanelSection::New => { + if has_param(params, "user") { + errors.push("client_new: must not include user".to_string()); + } + if get_param_value(params, "type").as_deref() != Some("m3u") { + errors.push("client_new: type must be m3u".to_string()); + } + } + PanelSection::Renew => { + if get_param_value(params, "type").as_deref() != Some("m3u") { + errors.push("client_renew: type must be m3u".to_string()); + } + if get_param_value(params, "username").as_deref() != Some("auto") { + errors.push("client_renew: username must be auto".to_string()); + } + if get_param_value(params, "password").as_deref() != Some("auto") { + errors.push("client_renew: password must be auto".to_string()); + } + } + PanelSection::Info => { + if get_param_value(params, "username").as_deref() != Some("auto") { + errors.push("client_info: username must be auto".to_string()); + } + if get_param_value(params, "password").as_deref() != Some("auto") { + errors.push("client_info: password must be auto".to_string()); + } + } + } + } + errors +} + +fn ensure_required_params(params: &mut Vec, section: PanelSection) { + let ensure = |params: &mut Vec, key: &str, value: &str| { + if !has_param(params, key) { + params.push(PanelApiQueryParamDto { + key: key.to_string(), + value: value.to_string(), + }); + } + }; + ensure(params, "api_key", "auto"); + match section { + PanelSection::New => { + params.retain(|p| !p.key.trim().eq_ignore_ascii_case("user")); + ensure(params, "type", "m3u"); + } + PanelSection::Renew => { + ensure(params, "type", "m3u"); + ensure(params, "username", "auto"); + ensure(params, "password", "auto"); + } + PanelSection::Info => { + ensure(params, "username", "auto"); + ensure(params, "password", "auto"); + } + } +} + +fn with_input_mut(form: &mut SourcesConfigDto, source_idx: usize, input_idx: usize, f: impl FnOnce(&mut ConfigInputDto)) { + if let Some(source) = form.sources.get_mut(source_idx) { + if let Some(input) = source.inputs.get_mut(input_idx) { + f(input); + } + } +} + +impl Reducible for PanelConfigFormState { + type Action = PanelConfigFormAction; + + fn reduce(self: Rc, action: Self::Action) -> Rc { + match action { + PanelConfigFormAction::SetAll(form) => Self { form, modified: false }.into(), + PanelConfigFormAction::SetEnabled { source_idx, input_idx, enabled } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + if enabled { + if input.panel_api.is_none() { + input.panel_api = Some(PanelApiConfigDto::default()); + } + } else { + input.panel_api = None; + } + }); + Self { form, modified: true }.into() + } + PanelConfigFormAction::SetPanelUrl { source_idx, input_idx, url } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + let panel = input.panel_api.get_or_insert_with(PanelApiConfigDto::default); + panel.url = url; + }); + Self { form, modified: true }.into() + } + PanelConfigFormAction::SetApiKey { source_idx, input_idx, api_key } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + let panel = input.panel_api.get_or_insert_with(PanelApiConfigDto::default); + panel.api_key = if api_key.trim().is_empty() { None } else { Some(api_key) }; + }); + Self { form, modified: true }.into() + } + PanelConfigFormAction::AddParam { source_idx, input_idx, section } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + let panel = input.panel_api.get_or_insert_with(PanelApiConfigDto::default); + params_mut(panel, section).push(PanelApiQueryParamDto { key: String::new(), value: String::new() }); + }); + Self { form, modified: true }.into() + } + PanelConfigFormAction::RemoveParam { source_idx, input_idx, section, param_idx } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + if let Some(panel) = input.panel_api.as_mut() { + let params = params_mut(panel, section); + if param_idx < params.len() { + params.remove(param_idx); + } + } + }); + Self { form, modified: true }.into() + } + PanelConfigFormAction::SetParamKey { source_idx, input_idx, section, param_idx, key } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + if let Some(panel) = input.panel_api.as_mut() { + let params = params_mut(panel, section); + if let Some(p) = params.get_mut(param_idx) { + p.key = key; + } + } + }); + Self { form, modified: true }.into() + } + PanelConfigFormAction::SetParamValue { source_idx, input_idx, section, param_idx, value } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + if let Some(panel) = input.panel_api.as_mut() { + let params = params_mut(panel, section); + if let Some(p) = params.get_mut(param_idx) { + p.value = value; + } + } + }); + Self { form, modified: true }.into() + } + PanelConfigFormAction::EnsureRequired { source_idx, input_idx, section } => { + let mut form = self.form.clone(); + with_input_mut(&mut form, source_idx, input_idx, |input| { + let panel = input.panel_api.get_or_insert_with(PanelApiConfigDto::default); + ensure_required_params(params_mut(panel, section), section); + }); + Self { form, modified: true }.into() + } + } + } +} + +fn render_param_editor( + form_state: &UseReducerHandle, + edit_mode: bool, + source_idx: usize, + input_idx: usize, + section: PanelSection, + section_title: String, + params: &[PanelApiQueryParamDto], +) -> Html { + let add_required = { + let form_state = form_state.clone(); + Callback::from(move |(name, _): (String, web_sys::MouseEvent)| { + if name == "required" { + form_state.dispatch(PanelConfigFormAction::EnsureRequired { source_idx, input_idx, section }); + } + }) + }; + + let add_param = { + let form_state = form_state.clone(); + Callback::from(move |(name, _): (String, web_sys::MouseEvent)| { + if name == "add" { + form_state.dispatch(PanelConfigFormAction::AddParam { source_idx, input_idx, section }); + } + }) + }; + + html! { +
+
+

{ section_title }

+ { html_if!(edit_mode, { +
+ + +
+ })} +
+
+ { + if params.is_empty() { + html!{
{ "—" }
} + } else { + html!{ for params.iter().enumerate().map(|(param_idx, p)| { + let on_key = { + let form_state = form_state.clone(); + Callback::from(move |value: String| { + form_state.dispatch(PanelConfigFormAction::SetParamKey { source_idx, input_idx, section, param_idx, key: value }); + }) + }; + let on_val = { + let form_state = form_state.clone(); + Callback::from(move |value: String| { + form_state.dispatch(PanelConfigFormAction::SetParamValue { source_idx, input_idx, section, param_idx, value }); + }) + }; + let on_remove = { + let form_state = form_state.clone(); + Callback::from(move |(name, _): (String, web_sys::MouseEvent)| { + if name == "rm" { + form_state.dispatch(PanelConfigFormAction::RemoveParam { source_idx, input_idx, section, param_idx }); + } + }) + }; + html!{ +
+ { + if edit_mode { + html!{ + <> + ::None} value={p.key.clone()} placeholder={Some("key".to_string())} on_change={Some(on_key)} /> + ::None} value={p.value.clone()} placeholder={Some("value".to_string())} on_change={Some(on_val)} /> + + + } + } else { + html!{ +
+ { &p.key } + { &p.value } +
+ } + } + } +
+ } + }) } + } + } +
+
+ } +} + +#[function_component] +pub fn PanelConfigView() -> Html { + let translate = use_translation(); + let config_ctx = use_context::().expect("ConfigContext not found"); + let config_view_ctx = use_context::().expect("ConfigViewContext not found"); + + let form_state: UseReducerHandle = use_reducer(|| PanelConfigFormState { + form: SourcesConfigDto::default(), + modified: false, + }); + + { + let on_form_change = config_view_ctx.on_form_change.clone(); + let deps = (form_state.clone(), form_state.modified); + use_effect_with(deps, move |(state, modified)| { + on_form_change.emit(ConfigForm::Panel(*modified, state.form.clone())); + }); + } + + { + let form_state = form_state.clone(); + let sources_cfg = config_ctx.config.as_ref().map(|c| c.sources.clone()); + use_effect_with((sources_cfg, config_view_ctx.edit_mode.clone()), move |(sources_cfg, _mode)| { + if let Some(src) = sources_cfg { + form_state.dispatch(PanelConfigFormAction::SetAll((*src).clone())); + } else { + form_state.dispatch(PanelConfigFormAction::SetAll(SourcesConfigDto::default())); + } + }); + } + + let render_input_card = |source_idx: usize, input_idx: usize, input: &ConfigInputDto| -> Html { + let panel_enabled = input.panel_api.is_some(); + let errors = validate_panel(input.panel_api.as_ref()); + let has_errors = !errors.is_empty(); + let status_chip = if panel_enabled { + if has_errors { + html! { } + } else { + html! { } + } + } else { + html! { ::None}/> } + }; + + let on_toggle = { + let form_state = form_state.clone(); + Callback::from(move |enabled: bool| { + form_state.dispatch(PanelConfigFormAction::SetEnabled { source_idx, input_idx, enabled }); + }) + }; + + let on_url = { + let form_state = form_state.clone(); + Callback::from(move |value: String| { + form_state.dispatch(PanelConfigFormAction::SetPanelUrl { source_idx, input_idx, url: value }); + }) + }; + + let on_api_key = { + let form_state = form_state.clone(); + Callback::from(move |value: String| { + form_state.dispatch(PanelConfigFormAction::SetApiKey { source_idx, input_idx, api_key: value }); + }) + }; + + let panel = input.panel_api.as_ref(); + let url_val = panel.map(|p| p.url.clone()).unwrap_or_default(); + let api_key_val = panel.and_then(|p| p.api_key.clone()).unwrap_or_default(); + let client_info = panel.map(|p| p.query_parameter.client_info.as_slice()).unwrap_or(&[]); + let client_new = panel.map(|p| p.query_parameter.client_new.as_slice()).unwrap_or(&[]); + let client_renew = panel.map(|p| p.query_parameter.client_renew.as_slice()).unwrap_or(&[]); + + html! { + +
+
+

{ &input.name }

+
{ format!("type={}", input.input_type) }
+
+
+ { status_chip } + { html_if!(*config_view_ctx.edit_mode, { +
+ { translate.t(LABEL_ENABLED) } + +
+ })} +
+
+ + { + if !panel_enabled { + html! {
{ translate.t(HINT_PANEL_ENABLE) }
} + } else if *config_view_ctx.edit_mode { + html! { + <> + + + { render_param_editor(&form_state, true, source_idx, input_idx, PanelSection::Info, translate.t(PanelSection::Info.label_key()), client_info) } + { render_param_editor(&form_state, true, source_idx, input_idx, PanelSection::New, translate.t(PanelSection::New.label_key()), client_new) } + { render_param_editor(&form_state, true, source_idx, input_idx, PanelSection::Renew, translate.t(PanelSection::Renew.label_key()), client_renew) } + { html_if!(has_errors, { +
+

{ translate.t(LABEL_VALIDATION) }

+
    + { for errors.iter().map(|e| html!{
  • { e }
    • }) } +
+
+ })} + + } + } else { + html!{ + <> +
+
+ + { if url_val.is_empty() { "—".to_string() } else { url_val } } +
+
+ + { if api_key_val.is_empty() { "—".to_string() } else { "••••••••".to_string() } } +
+
+ + { format!("{} params", client_info.len()) } +
+
+ + { format!("{} params", client_new.len()) } +
+
+ + { format!("{} params", client_renew.len()) } +
+
+ { html_if!(has_errors, { +
+

{ translate.t(LABEL_VALIDATION) }

+
    + { for errors.iter().map(|e| html!{
  • { e }
    • }) } +
+
+ })} + + } + } + } + + } + }; + + let inputs = form_state + .form + .sources + .iter() + .enumerate() + .flat_map(|(sidx, s)| s.inputs.iter().enumerate().map(move |(iidx, inp)| (sidx, iidx, inp))) + .collect::>(); + + html! { +
+
{ translate.t(LABEL_PANEL_CONFIG) }
+
+ +

{ translate.t(LABEL_PANEL_CONFIG) }

+
+ { translate.t(HINT_PANEL_INFO) } +
+ +
+
+ { for inputs.into_iter().map(|(sidx, iidx, inp)| render_input_card(sidx, iidx, inp)) } +
+
+ } +} diff --git a/frontend/src/app/components/userlist/user_table.rs b/frontend/src/app/components/userlist/user_table.rs index ea821d92f..0099f68f9 100644 --- a/frontend/src/app/components/userlist/user_table.rs +++ b/frontend/src/app/components/userlist/user_table.rs @@ -197,9 +197,8 @@ pub fn UserTable(props: &UserTableProps) -> Html { }) }; - let is_sortable = Callback::::from(move |col| { - is_col_sortable(col) - }); + + let is_sortable = Callback::::from(is_col_sortable); let on_sort = { let users = props.users.clone(); diff --git a/frontend/src/services/config_service.rs b/frontend/src/services/config_service.rs index 41cbbd075..78ac89985 100644 --- a/frontend/src/services/config_service.rs +++ b/frontend/src/services/config_service.rs @@ -1,6 +1,6 @@ use crate::model::WebConfig; use crate::services::{get_base_href, request_get, request_post, EventService}; -use shared::model::{AppConfigDto, ConfigDto, ConfigInputDto, IpCheckDto, LibraryScanRequest, TargetOutputDto}; +use shared::model::{AppConfigDto, ConfigDto, ConfigInputDto, IpCheckDto, LibraryScanRequest, SourcesConfigDto, TargetOutputDto}; use std::cell::RefCell; use std::future::Future; use std::rc::Rc; @@ -19,6 +19,7 @@ pub struct ConfigService { config_channel: Mutable>>, is_fetching: AtomicBool, config_path: String, + sources_path: String, ip_check_path: String, batch_input_content_path: String, geoip_path: String, @@ -29,12 +30,14 @@ pub struct ConfigService { impl ConfigService { pub fn new(config: &WebConfig, event_service: Rc) -> Self { let base_href = get_base_href(); + let config_path = concat_path_leading_slash(&base_href, "api/v1/config"); Self { ui_config: Rc::new(config.clone()), server_config: RefCell::new(None), config_channel: Mutable::new(None), is_fetching: AtomicBool::new(false), - config_path: concat_path_leading_slash(&base_href, "api/v1/config"), + config_path: config_path.clone(), + sources_path: concat_path(&config_path, "sources"), ip_check_path: concat_path_leading_slash(&base_href, "api/v1/ipinfo"), batch_input_content_path: concat_path_leading_slash(&base_href, "api/v1/config/batchContent"), geoip_path: concat_path_leading_slash(&base_href, "api/v1/geoip/update"), @@ -146,6 +149,21 @@ impl ConfigService { } } + pub async fn save_sources(&self, dto: SourcesConfigDto) -> Result<(), Error> { + self.event_service.set_config_change_message_blocked(true); + match request_post::(&self.sources_path, dto, None, None).await { + Ok(_) => { + self.event_service.set_config_change_message_blocked(false); + Ok(()) + } + Err(err) => { + self.event_service.set_config_change_message_blocked(false); + error!("{err}"); + Err(err) + } + } + } + pub async fn update_geoip(&self) -> Result, Error> { request_get::<()>(&self.geoip_path, None, None).await } diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 03e48aa3d..3fe60c082 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "shared" -version = "3.2.21" +version = "3.2.22" edition = "2021" +rust-version = "1.87.0" [dependencies] bitflags = "2" diff --git a/shared/src/model/config/input.rs b/shared/src/model/config/input.rs index 78f489c7a..23183f149 100644 --- a/shared/src/model/config/input.rs +++ b/shared/src/model/config/input.rs @@ -1,6 +1,7 @@ use crate::error::{TuliproxError, TuliproxErrorKind}; use crate::model::{EpgConfigDto}; use crate::utils::{default_as_true, get_credentials_from_url_str, get_trimmed_string, sanitize_sensitive_info, trim_last_slash, deserialize_timestamp}; +use super::PanelApiConfigDto; use crate::{check_input_credentials, check_input_connections, create_tuliprox_error_result, handle_tuliprox_error_result_list, info_err}; use enum_iterator::Sequence; use std::collections::{HashMap, HashSet}; @@ -307,6 +308,8 @@ pub struct ConfigInputDto { pub staged: Option, #[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")] pub exp_date: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub panel_api: Option, } impl Default for ConfigInputDto { @@ -329,6 +332,7 @@ impl Default for ConfigInputDto { method: InputFetchMethod::default(), staged: None, exp_date: None, + panel_api: None, } } } diff --git a/shared/src/model/config/library.rs b/shared/src/model/config/library.rs index d536b4e86..4f5b27ac8 100644 --- a/shared/src/model/config/library.rs +++ b/shared/src/model/config/library.rs @@ -16,7 +16,7 @@ fn default_tmdb_language() -> String { "en-US".to_string() } fn default_storage_formats() -> Vec { - vec![LibraryMetadataFormat::Nfo] + vec![] } fn default_movie_category() -> String { "Local Movies".to_string() diff --git a/shared/src/model/config/mod.rs b/shared/src/model/config/mod.rs index 318470115..8ee36bc6f 100644 --- a/shared/src/model/config/mod.rs +++ b/shared/src/model/config/mod.rs @@ -12,6 +12,7 @@ mod video_download; mod schedule; mod log; mod input; +mod panel_api; mod stream; mod epg; mod reverse_proxy; @@ -54,6 +55,7 @@ pub use video_download::*; pub use schedule::*; pub use log::*; pub use input::*; +pub use panel_api::*; pub use stream::*; pub use epg_smart_match::*; pub use epg::*; @@ -71,4 +73,4 @@ pub use config_type::*; pub use playlist_update_state::*; pub use favourites::*; pub use library::*; -pub use crate::apply_batch_aliases; \ No newline at end of file +pub use crate::apply_batch_aliases; diff --git a/shared/src/model/config/panel_api.rs b/shared/src/model/config/panel_api.rs new file mode 100644 index 000000000..52fdbd42d --- /dev/null +++ b/shared/src/model/config/panel_api.rs @@ -0,0 +1,31 @@ +use crate::utils::{default_as_true}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct PanelApiQueryParamDto { + pub key: String, + pub value: String, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct PanelApiQueryParametersDto { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub client_info: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub client_new: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub client_renew: Vec, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct PanelApiConfigDto { + #[serde(default = "default_as_true")] + pub enabled: bool, + pub url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub api_key: Option, + #[serde(default)] + pub query_parameter: PanelApiQueryParametersDto, +} diff --git a/shared/src/utils/constants.rs b/shared/src/utils/constants.rs index ad8e9268e..58c059c58 100644 --- a/shared/src/utils/constants.rs +++ b/shared/src/utils/constants.rs @@ -96,7 +96,6 @@ pub struct Constants { pub allowed_output_formats: Vec, pub re_trakt_year: Regex, pub re_quality: Regex, - pub re_classifier_quality: Regex, pub re_classifier_year: Regex, pub re_classifier_cleanup: Regex, pub re_classifier_episode: Regex, @@ -154,8 +153,7 @@ pub static CONSTANTS: LazyLock = LazyLock::new(|| "ye", "zm", "zw", ].into_iter().collect::>(), re_trakt_year: Regex::new(r"\(?(\d{4})\)?$").unwrap(), - re_quality: Regex::new(r"(?i)\b(4K|UHD|8K|2160p?|1080p?|720p?|480p?|BLURAY|HDTV|DVDRIP|CAM|TS|HDR|DV|SDR)\b").unwrap(), - re_classifier_quality: Regex::new(r"(?i)[\s\._-]*(1080p|720p|480p|2160p|4K|BluRay|BRRip|WEB-DL|WEBRip|HDTV|DVDRip|CAM|TS|HDR|DV|SDR|UHD|8K).*$").unwrap(), + re_quality: Regex::new(r"(?i)\b(x265|4K|UHD|8K|2160p?|1080p?|720p?|480p?|BLURAY|HDTV|DVDRIP|BRRIP|CAM|TS|HDR|DV|SDR)\b").unwrap(), re_classifier_year: Regex::new(r"[\(\[]?(\d{4})[\)\]]?").unwrap(), re_classifier_cleanup: Regex::new(r"(?i)[\s\._-]*(?:s\d+e\d+|\d+x\d+|season[\s\._-]*\d+|episode[\s\._-]*\d+).*$").unwrap(), re_classifier_episode: Regex::new(r"(?i)(?:e|episode|x)[\s\._-]*(\d+)").unwrap(),