euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-12 19:02:19 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

Feature/staged input and path improvements (#632)

Browse Source 
* No template file not found error message
* Setup wizard improvmenets
* path improvements
* staged input fix
* path improvements
* Provider dns resolved ips now writing to a file instead of source.yml
* Playlist update hard kill after 1h
* matadata update tmdb fix
* Web UI fixes
* GeoIp update schedule
* batch:// prefix  for aliases
This commit is contained in:
euzu
2026-03-06 17:29:42 +01:00
committed by GitHub
parent c5632d0c98
commit 737d04d08b
82 changed files with 2506 additions and 1433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/api/config_file.rs
+13 -9
View File
@@ -223,20 +223,24 @@ impl ConfigFile {
// ── Parse ────────────────────────────────────────────────────
let config_dto = read_config_file(config_file.as_str(), true, true)?;
let default_mapping_path = utils::get_default_mappings_path(paths.config_path.as_str());
let current_mapping_path = paths.mapping_file_path.clone().unwrap_or_else(|| default_mapping_path.clone());
let next_mapping_path =
config_dto.mapping_path.clone().filter(|path| !path.trim().is_empty()).unwrap_or(default_mapping_path);
let current_mapping_path = paths.mapping_file_path.clone().unwrap_or_else(|| {
utils::resolve_mapping_file_path(paths.config_path.as_str(), None)
});
let next_mapping_path = paths.mapping_file_path.clone().unwrap_or_else(|| {
utils::resolve_mapping_file_path(paths.config_path.as_str(), config_dto.mapping_path.as_deref())
});
let mapping_changed = current_mapping_path != next_mapping_path;
let default_template_path = utils::get_default_templates_path(paths.config_path.as_str());
let current_template_path = paths.template_file_path.clone().unwrap_or_else(|| default_template_path.clone());
let next_template_path =
config_dto.template_path.clone().filter(|path| !path.trim().is_empty()).unwrap_or(default_template_path);
let current_template_path = paths.template_file_path.clone().unwrap_or_else(|| {
utils::resolve_template_file_path(paths.config_path.as_str(), None)
});
let next_template_path = paths.template_file_path.clone().unwrap_or_else(|| {
utils::resolve_template_file_path(paths.config_path.as_str(), config_dto.template_path.as_deref())
});
let template_changed = current_template_path != next_template_path;
let mut config: Config = Config::from(config_dto);
config.prepare(paths.config_path.as_str())?;
config.prepare(paths.config_path.as_str(), paths.home_path.as_str())?;
// Compute effective runtime paths for the NEW config before apply.
// This ensures prepare-phase reads/validates against the same files that will be active after apply.
backend/src/api/endpoints/api_playlist_utils.rs
+1 -1
View File
@@ -129,7 +129,7 @@ pub(in crate::api::endpoints) async fn get_playlist_for_custom_provider(
let cfg = app_state.app_config.config.load();
match cfg_input {
Some(input) => {
let (result, errors) = match input.input_type {
let (result, errors) = match input.get_download_input_type() {
InputType::M3u | InputType::M3uBatch => {
m3u::download_m3u_playlist(&app_state.app_config, client, &cfg, input).await
}
backend/src/api/endpoints/library_api.rs
+5 -5
View File
@@ -34,10 +34,10 @@ async fn scan_library(
};
// Check if Library is enabled
let (lib_config, metadata_update_config, working_dir) = {
let (lib_config, metadata_update_config, storage_dir) = {
let config = app_state.app_config.config.load();
match config.library.as_ref() {
Some(lib) if lib.enabled => (lib.clone(), config.metadata_update.clone(), config.working_dir.clone()),
Some(lib) if lib.enabled => (lib.clone(), config.metadata_update.clone(), config.storage_dir.clone()),
_ => {
let response = LibraryScanSummary {
status: LibraryScanSummaryStatus::Error,
@@ -61,7 +61,7 @@ async fn scan_library(
lib_config,
metadata_update_config,
client,
LibraryScanTaskOptions { force_rescan: request.force_rescan, message_prefix: "", working_dir },
LibraryScanTaskOptions { force_rescan: request.force_rescan, message_prefix: "", storage_dir },
permit,
);
@@ -78,7 +78,7 @@ async fn get_library_status(
let client = app_state.http_client.load_full().as_ref().clone();
// Get statistics from processor
let processor =
LibraryProcessor::new(config.clone(), config_snapshot.metadata_update.as_ref(), client, &config_snapshot.working_dir);
LibraryProcessor::new(config.clone(), config_snapshot.metadata_update.as_ref(), client, &config_snapshot.storage_dir);
let entries = processor.get_all_entries().await;
let movies = entries.iter().filter(|e| e.metadata.is_movie()).count();
@@ -90,7 +90,7 @@ async fn get_library_status(
movies,
series,
path: Some(
resolve_metadata_storage_path(config_snapshot.metadata_update.as_ref(), &config_snapshot.working_dir)
resolve_metadata_storage_path(config_snapshot.metadata_update.as_ref(), &config_snapshot.storage_dir)
.to_string_lossy()
.to_string(),
),
backend/src/api/endpoints/v1_api.rs
+13 -66
View File
@@ -10,22 +10,16 @@ use crate::{
model::AppState,
},
auth::validator_admin,
model::InputSource,
repository::get_geoip_path,
utils::{ip_checker::get_ips, request::download_text_content, GeoIp},
processing::geoip::{update_geoip_db, GeoIpUpdateError},
utils::ip_checker::get_ips,
VERSION,
};
use axum::response::IntoResponse;
use log::{error, info};
use shared::{
model::{default_geoip_url, InputFetchMethod, IpCheckDto, StatusCheck},
utils::{concat_path_leading_slash, Internable},
};
use std::{
collections::{BTreeMap, HashMap},
io::Cursor,
sync::Arc,
model::{IpCheckDto, StatusCheck},
utils::concat_path_leading_slash,
};
use std::{collections::BTreeMap, sync::Arc};
async fn create_ipinfo_check(app_state: &Arc<AppState>) -> Option<(Option<String>, Option<String>)> {
let config = app_state.app_config.config.load();
@@ -85,63 +79,16 @@ async fn streams(
async fn geoip_update(
axum::extract::State(app_state): axum::extract::State<Arc<AppState>>,
) -> axum::response::Response {
let config = app_state.app_config.config.load();
if let Some(geoip) = config.reverse_proxy.as_ref().and_then(|r| r.geoip.as_ref()) {
if geoip.enabled {
let geoip_db_path = &*get_geoip_path(&config.working_dir);
let _file_lock = app_state.app_config.file_locks.write_lock(geoip_db_path).await;
let url = if geoip.url.trim().is_empty() { default_geoip_url() } else { geoip.url.clone() };
let input_source = InputSource {
name: "GeoIP".intern(),
url,
provider: None,
username: None,
password: None,
method: InputFetchMethod::GET,
headers: HashMap::default(),
};
return match download_text_content(
&app_state.app_config,
&app_state.http_client.load(),
&input_source,
None,
None,
false,
)
.await
{
Ok((content, _)) => {
let reader = Cursor::new(content);
let mut geoip = GeoIp::new();
let result = {
match geoip.import_ipv4_from_csv(reader, geoip_db_path) {
Ok(size) => (Some(size), None),
Err(err) => (None, Some(err)),
}
};
return match result {
(Some(_), None) => {
info!("GeoIp db updated");
app_state.geoip.store(Some(Arc::new(geoip)));
axum::http::StatusCode::OK.into_response()
}
(None, Some(err)) => {
error!("Failed to process geoip db: {err}");
internal_server_error!()
}
_ => internal_server_error!(),
};
}
Err(err) => {
error!("Failed to download geoip db: {err}");
axum::http::StatusCode::BAD_REQUEST.into_response()
}
};
match update_geoip_db(&app_state).await {
Ok(()) => axum::http::StatusCode::OK.into_response(),
Err(GeoIpUpdateError::Disabled | GeoIpUpdateError::DownloadFailed(_)) => {
axum::http::StatusCode::BAD_REQUEST.into_response()
}
Err(err) => {
log::error!("GeoIp update failed: {err}");
internal_server_error!()
}
}
axum::http::StatusCode::BAD_REQUEST.into_response()
}
async fn ipinfo(axum::extract::State(app_state): axum::extract::State<Arc<AppState>>) -> axum::response::Response {
backend/src/api/endpoints/v1_api_config.rs
+4 -370
View File
@@ -16,11 +16,11 @@ use axum::{
response::IntoResponse,
Router,
};
use log::{error, warn};
use log::error;
use serde_json::json;
use shared::{
error::TuliproxError,
model::{ApiProxyConfigDto, ConfigDto, ConfigProviderDto, ProviderDnsDto, SourcesConfigDto},
model::{ApiProxyConfigDto, ConfigDto, SourcesConfigDto},
utils::{
HEADER_CONFIG_API_PROXY_REVISION, HEADER_CONFIG_MAIN_REVISION, HEADER_CONFIG_SOURCES_REVISION, HEADER_IF_MATCH,
},
@@ -28,112 +28,7 @@ use shared::{
use std::path::Path;
use std::sync::Arc;
fn inject_provider_dns_resolved(sources_dto: &mut SourcesConfigDto, runtime_sources: &crate::model::SourcesConfig) {
let Some(provider_dtos) = sources_dto.provider.as_mut() else {
return;
};
for provider_dto in provider_dtos {
let Some(runtime_provider) = runtime_sources.get_provider_by_name(provider_dto.name.as_ref()) else {
continue;
};
let Some(dns_dto) = provider_dto.dns.as_mut() else {
continue;
};
if runtime_provider.get_dns_config().is_none() {
dns_dto.resolved = None;
continue;
}
let snapshot = runtime_provider.snapshot_resolved_ordered();
dns_dto.resolved = (!snapshot.is_empty()).then_some(snapshot);
}
}
fn strip_provider_dns_resolved(sources_dto: &mut SourcesConfigDto) {
let Some(provider_dtos) = sources_dto.provider.as_mut() else {
return;
};
for provider_dto in provider_dtos {
if let Some(dns_dto) = provider_dto.dns.as_mut() {
dns_dto.resolved = None;
}
}
}
fn merge_provider_dns_resolved_from_existing(
sources_dto: &mut SourcesConfigDto,
existing_sources_dto: &SourcesConfigDto,
) {
let Some(provider_dtos) = sources_dto.provider.as_mut() else {
return;
};
let Some(existing_provider_dtos) = existing_sources_dto.provider.as_ref() else {
return;
};
let mut resolved_by_provider_name = std::collections::HashMap::new();
for provider in existing_provider_dtos {
let Some(dns) = provider.dns.as_ref() else {
continue;
};
let Some(resolved) = dns.resolved.as_ref() else {
continue;
};
if !resolved.is_empty() {
resolved_by_provider_name.insert(provider.name.to_string(), resolved.clone());
}
}
if resolved_by_provider_name.is_empty() {
return;
}
for provider in provider_dtos {
let Some(dns) = provider.dns.as_mut() else {
continue;
};
if !dns.enabled {
continue;
}
if dns.resolved.is_some() {
continue;
}
if let Some(resolved) = resolved_by_provider_name.get(provider.name.as_ref()) {
dns.resolved = Some(resolved.clone());
}
}
}
fn build_existing_sources_from_runtime(runtime_sources: &crate::model::SourcesConfig) -> Option<SourcesConfigDto> {
let providers: Vec<ConfigProviderDto> = runtime_sources
.provider
.iter()
.filter_map(|runtime_provider| {
let resolved = runtime_provider.snapshot_resolved_ordered();
if resolved.is_empty() {
return None;
}
Some(ConfigProviderDto {
name: runtime_provider.name.clone(),
urls: runtime_provider.urls.clone(),
dns: Some(ProviderDnsDto {
enabled: runtime_provider.get_dns_config().is_some_and(|cfg| cfg.enabled),
resolved: Some(resolved),
..ProviderDnsDto::default()
}),
})
})
.collect();
if providers.is_empty() {
None
} else {
Some(SourcesConfigDto {
provider: Some(providers),
..SourcesConfigDto::default()
})
}
}
fn file_revision_from_bytes(bytes: &[u8]) -> String { blake3::hash(bytes).to_hex().to_string() }
@@ -273,7 +168,7 @@ async fn save_config_main(
async fn save_config_sources(
axum::extract::State(app_state): axum::extract::State<Arc<AppState>>,
headers: HeaderMap,
axum::extract::Json(mut sources): axum::extract::Json<SourcesConfigDto>,
axum::extract::Json(sources): axum::extract::Json<SourcesConfigDto>,
) -> impl axum::response::IntoResponse + Send {
let sources_file_path = {
let paths = app_state.app_config.paths.load();
@@ -294,27 +189,6 @@ async fn save_config_sources(
return response;
}
let existing_sources_dto =
match utils::read_sources_file_from_path(Path::new(&sources_file_path), false, false, None).await {
Ok(existing) => Some(existing),
Err(err) => {
warn!("Failed to preload existing source.yml from '{sources_file_path}' before save: {err}");
let runtime_sources = app_state.app_config.sources.load();
let runtime_fallback = build_existing_sources_from_runtime(runtime_sources.as_ref());
if runtime_fallback.is_some() {
warn!("Using runtime provider dns snapshot as fallback for preserving dns.resolved");
}
runtime_fallback
}
};
// `dns.resolved` is runtime-managed and must not be accepted from API input,
// but existing runtime values should survive UI saves until the next DNS refresh updates them.
strip_provider_dns_resolved(&mut sources);
if let Some(existing_sources_dto) = existing_sources_dto.as_ref() {
merge_provider_dns_resolved_from_existing(&mut sources, existing_sources_dto);
}
let templates_to_persist = match utils::validate_source_config_for_persist(&app_state, &sources).await {
Ok(value) => value,
Err(err) => {
@@ -499,8 +373,6 @@ async fn config(axum::extract::State(app_state): axum::extract::State<Arc<AppSta
error!("Failed to prepare users: {err}");
internal_server_error!()
} else {
let runtime_sources = app_state.app_config.sources.load();
inject_provider_dns_resolved(&mut app_config.sources, &runtime_sources);
let response = axum::response::Json(app_config).into_response();
let response = response_with_revision_header(response, HEADER_CONFIG_MAIN_REVISION, &main_revision);
let response = response_with_revision_header(response, HEADER_CONFIG_SOURCES_REVISION, &sources_revision);
@@ -562,247 +434,9 @@ pub fn v1_api_config_register(router: Router<Arc<AppState>>) -> axum::Router<Arc
#[cfg(test)]
mod tests {
use super::{
build_existing_sources_from_runtime, inject_provider_dns_resolved, merge_provider_dns_resolved_from_existing,
require_matching_revision, strip_provider_dns_resolved,
};
use super::require_matching_revision;
use axum::http::{HeaderMap, HeaderValue, StatusCode};
use crate::model::{ConfigProvider, SourcesConfig};
use indexmap::IndexMap;
use shared::model::{ConfigProviderDto, DnsScheme, ProviderDnsDto, SourcesConfigDto};
use shared::utils::{HEADER_CONFIG_SOURCES_REVISION, HEADER_IF_MATCH};
use std::{net::IpAddr, sync::Arc};
#[test]
fn inject_provider_dns_resolved_populates_runtime_snapshot() {
let mut dto = SourcesConfigDto {
provider: Some(vec![ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
schemes: Some(vec![DnsScheme::Http]),
..ProviderDnsDto::default()
}),
}]),
..SourcesConfigDto::default()
};
let runtime_provider = Arc::new(ConfigProvider::from(&ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
schemes: Some(vec![DnsScheme::Http]),
..ProviderDnsDto::default()
}),
}));
runtime_provider.store_resolved(
"example.com",
vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")],
);
let runtime_sources = SourcesConfig {
provider: vec![runtime_provider],
..SourcesConfig::default()
};
inject_provider_dns_resolved(&mut dto, &runtime_sources);
let resolved = dto.provider.as_ref()
.and_then(|providers| providers.first())
.and_then(|provider| provider.dns.as_ref())
.and_then(|dns| dns.resolved.as_ref())
.expect("resolved dns snapshot should be present");
assert_eq!(
resolved.get("example.com"),
Some(&vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")])
);
}
#[test]
fn inject_provider_dns_resolved_clears_value_when_runtime_dns_disabled() {
let mut dto = SourcesConfigDto {
provider: Some(vec![ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
resolved: Some(IndexMap::from([(
"example.com".to_string(),
vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")],
)])),
..ProviderDnsDto::default()
}),
}]),
..SourcesConfigDto::default()
};
let runtime_provider = Arc::new(ConfigProvider::from(&ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: None,
}));
let runtime_sources = SourcesConfig {
provider: vec![runtime_provider],
..SourcesConfig::default()
};
inject_provider_dns_resolved(&mut dto, &runtime_sources);
let resolved = dto.provider.as_ref()
.and_then(|providers| providers.first())
.and_then(|provider| provider.dns.as_ref())
.and_then(|dns| dns.resolved.as_ref());
assert!(resolved.is_none(), "resolved output must be empty when runtime dns is disabled");
}
#[test]
fn strip_provider_dns_resolved_removes_payload_values() {
let mut dto = SourcesConfigDto {
provider: Some(vec![ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
resolved: Some(IndexMap::from([(
"example.com".to_string(),
vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")],
)])),
..ProviderDnsDto::default()
}),
}]),
..SourcesConfigDto::default()
};
strip_provider_dns_resolved(&mut dto);
let resolved = dto
.provider
.as_ref()
.and_then(|providers| providers.first())
.and_then(|provider| provider.dns.as_ref())
.and_then(|dns| dns.resolved.as_ref());
assert!(resolved.is_none(), "resolved must be stripped from incoming payload");
}
#[test]
fn merge_provider_dns_resolved_from_existing_preserves_runtime_snapshot() {
let mut incoming = SourcesConfigDto {
provider: Some(vec![ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
..ProviderDnsDto::default()
}),
}]),
..SourcesConfigDto::default()
};
let existing = SourcesConfigDto {
provider: Some(vec![ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
resolved: Some(IndexMap::from([(
"example.com".to_string(),
vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")],
)])),
..ProviderDnsDto::default()
}),
}]),
..SourcesConfigDto::default()
};
merge_provider_dns_resolved_from_existing(&mut incoming, &existing);
let resolved = incoming
.provider
.as_ref()
.and_then(|providers| providers.first())
.and_then(|provider| provider.dns.as_ref())
.and_then(|dns| dns.resolved.as_ref())
.expect("resolved should be merged from existing source dto");
assert_eq!(
resolved.get("example.com"),
Some(&vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")])
);
}
#[test]
fn merge_provider_dns_resolved_from_existing_skips_disabled_dns() {
let mut incoming = SourcesConfigDto {
provider: Some(vec![ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: false,
..ProviderDnsDto::default()
}),
}]),
..SourcesConfigDto::default()
};
let existing = SourcesConfigDto {
provider: Some(vec![ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
resolved: Some(IndexMap::from([(
"example.com".to_string(),
vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")],
)])),
..ProviderDnsDto::default()
}),
}]),
..SourcesConfigDto::default()
};
merge_provider_dns_resolved_from_existing(&mut incoming, &existing);
let resolved = incoming
.provider
.as_ref()
.and_then(|providers| providers.first())
.and_then(|provider| provider.dns.as_ref())
.and_then(|dns| dns.resolved.as_ref());
assert!(resolved.is_none(), "resolved should not be merged for disabled dns entries");
}
#[test]
fn build_existing_sources_from_runtime_exports_provider_dns_resolved() {
let runtime_provider = Arc::new(ConfigProvider::from(&ConfigProviderDto {
name: "p1".into(),
urls: vec!["http://example.com".into()],
dns: Some(ProviderDnsDto {
enabled: true,
schemes: Some(vec![DnsScheme::Http]),
..ProviderDnsDto::default()
}),
}));
runtime_provider.store_resolved(
"example.com",
vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")],
);
let runtime_sources = SourcesConfig {
provider: vec![runtime_provider],
..SourcesConfig::default()
};
let exported =
build_existing_sources_from_runtime(&runtime_sources).expect("runtime fallback dto should exist");
let resolved = exported
.provider
.as_ref()
.and_then(|providers| providers.first())
.and_then(|provider| provider.dns.as_ref())
.and_then(|dns| dns.resolved.as_ref())
.expect("resolved should be present in runtime fallback dto");
assert_eq!(
resolved.get("example.com"),
Some(&vec!["203.0.113.10".parse::<IpAddr>().expect("ip parse should work")])
);
}
#[test]
fn require_matching_revision_rejects_missing_if_match_header() {
backend/src/api/library_scan.rs
+3 -3
View File
@@ -10,7 +10,7 @@ use std::sync::Arc;
pub(crate) struct LibraryScanTaskOptions {
pub force_rescan: bool,
pub message_prefix: &'static str,
pub working_dir: String,
pub storage_dir: String,
}
pub(crate) fn spawn_library_scan(
@@ -21,11 +21,11 @@ pub(crate) fn spawn_library_scan(
options: LibraryScanTaskOptions,
permit: UpdateGuardPermit,
) {
let LibraryScanTaskOptions { force_rescan, message_prefix, working_dir } = options;
let LibraryScanTaskOptions { force_rescan, message_prefix, storage_dir } = options;
let prefix = message_prefix.to_string();
tokio::spawn(async move {
let _permit = permit;
let processor = LibraryProcessor::new(lib_config, metadata_update_config.as_ref(), client, &working_dir);
let processor = LibraryProcessor::new(lib_config, metadata_update_config.as_ref(), client, &storage_dir);
match processor.scan(force_rescan).await {
Ok(result) => {
info!("{prefix}Library scan completed successfully");
backend/src/api/main_api.rs
+1 -1
View File
@@ -204,7 +204,7 @@ async fn create_shared_data(
let use_geoip = config.is_geoip_enabled();
let geoip = if use_geoip {
let path = get_geoip_path(&config.working_dir);
let path = get_geoip_path(&config.storage_dir);
let _file_lock = app_config.file_locks.read_lock(&path).await;
match GeoIp::load(&path) {
Ok(db) => {
backend/src/api/model/active_provider_manager.rs
+2
View File
@@ -708,7 +708,9 @@ mod tests {
api_proxy: Arc::new(ArcSwapOption::default()),
file_locks: Arc::new(FileLockManager::default()),
paths: Arc::new(ArcSwap::from_pointee(ConfigPaths {
home_path: String::new(),
config_path: String::new(),
storage_path: String::new(),
config_file_path: String::new(),
sources_file_path: String::new(),
mapping_file_path: None,
backend/src/api/model/app_state.rs
+96 -17
View File
@@ -266,7 +266,7 @@ fn build_http_client_with_fallback(
pub fn create_cache(config: &Config) -> Option<Arc<Mutex<LRUResourceCache>>> {
let lru_cache = config.reverse_proxy.as_ref().and_then(|r| r.cache.as_ref()).and_then(|c| {
if c.enabled {
Some(LRUResourceCache::new(c.size, c.dir.as_str()))
Some(LRUResourceCache::new(c.size, c.directory.as_str()))
} else {
None
}
@@ -341,20 +341,23 @@ pub struct AppState {
impl AppState {
pub(in crate::api::model) async fn set_config(&self, config: Config) -> Result<UpdateChanges, TuliproxError> {
let old_storage_dir = self.app_config.config.load().storage_dir.clone();
let changes = self.detect_changes_for_config(&config);
config.update_runtime();
let use_geoip = config.is_geoip_enabled();
let working_dir = config.working_dir.clone();
let storage_dir = config.storage_dir.clone();
self.active_users.update_config(&config);
self.app_config.set_config(config)?;
self.active_provider.update_config(&self.app_config).await;
self.update_config().await?;
if changes.flags.contains(UpdateChangesFlags::Geoip) {
let geoip_reload_needed =
changes.flags.contains(UpdateChangesFlags::Geoip) || (use_geoip && old_storage_dir != storage_dir);
if geoip_reload_needed {
let new_geoip = if use_geoip {
let path = get_geoip_path(&working_dir);
let path = get_geoip_path(&storage_dir);
let _file_lock = self.app_config.file_locks.read_lock(&path).await;
GeoIp::load(&path).ok().map(Arc::new)
} else {
@@ -381,7 +384,7 @@ impl AppState {
.reverse_proxy
.as_ref()
.and_then(|r| r.cache.as_ref())
.map_or((false, 0, ""), |c| (c.enabled, c.size, c.dir.as_str()));
.map_or((false, 0, ""), |c| (c.enabled, c.size, c.directory.as_str()));
if let Some(cache) = self.cache.load().as_ref() {
if enabled {
@@ -401,6 +404,22 @@ impl AppState {
sources: SourcesConfig,
) -> Result<UpdateChanges, TuliproxError> {
let changes = self.detect_changes_for_sources(&sources);
// Carry over DNS caches from old providers so resolved IPs survive hot-reloads
// without waiting for the background resolver or the persisted-file seed.
{
let old_sources = self.app_config.sources.load();
for new_provider in &sources.provider {
if let Some(old_provider) = old_sources.get_provider_by_name(&new_provider.name) {
if new_provider.get_dns_config().is_some_and(|cfg| cfg.enabled) {
for (host, ips) in old_provider.snapshot_resolved() {
if !ips.is_empty() && new_provider.dns_cache.ip_count(&host) == 0 {
new_provider.dns_cache.store_resolved(&host, ips);
}
}
}
}
}
}
self.app_config.set_sources(sources)?;
self.active_provider.update_config(&self.app_config).await;
@@ -420,6 +439,9 @@ impl AppState {
let old_config = self.app_config.config.load();
let changed_schedules =
change_detect!(schedules_changed, old_config.schedules.as_ref(), config.schedules.as_ref());
let library_enabled = config.library.as_ref().is_some_and(|library| library.enabled);
let old_library_enabled = old_config.library.as_ref().is_some_and(|library| library.enabled);
let changed_library_enabled = library_enabled != old_library_enabled;
let changed_hdhomerun =
change_detect!(hdhomerun_changed, old_config.hdhomerun.as_ref(), config.hdhomerun.as_ref());
let changed_file_watch =
@@ -430,7 +452,7 @@ impl AppState {
let geoip_enabled_old = old_config.is_geoip_enabled();
let mut changes = UpdateChanges { flags: UpdateChangesFlagsSet::new(), targets: None };
changes.set_flag_if(changed_schedules, UpdateChangesFlags::Scheduler);
changes.set_flag_if(changed_schedules || changed_library_enabled || geoip_enabled != geoip_enabled_old, UpdateChangesFlags::Scheduler);
changes.set_flag_if(changed_hdhomerun, UpdateChangesFlags::Hdhomerun);
changes.set_flag_if(changed_file_watch, UpdateChangesFlags::FileWatch);
changes.set_flag_if(geoip_enabled != geoip_enabled_old, UpdateChangesFlags::Geoip);
@@ -582,19 +604,30 @@ fn schedules_changed(a: &[ScheduleConfig], b: &[ScheduleConfig]) -> bool {
if a.len() != b.len() {
return true;
}
let mut used = vec![false; b.len()];
for schedule in a {
let Some(found) = b.iter().find(|&s| s.schedule == schedule.schedule) else {
let Some(found_idx) = b.iter().enumerate().find_map(|(idx, candidate)| {
if used[idx]
|| candidate.schedule != schedule.schedule
|| candidate.task_type != schedule.task_type
{
return None;
}
let targets_match = match (schedule.targets.as_ref(), candidate.targets.as_ref()) {
(None, None) => true,
(Some(_), None) | (None, Some(_)) => false,
(Some(a_targets), Some(b_targets)) => small_vecs_equal_unordered(a_targets, b_targets),
};
if targets_match {
Some(idx)
} else {
None
}
}) else {
return true;
};
match (schedule.targets.as_ref(), found.targets.as_ref()) {
(None, None) => {}
(Some(_), None) | (None, Some(_)) => return true,
(Some(a_targets), Some(b_targets)) => {
if !small_vecs_equal_unordered(a_targets, b_targets) {
return true;
}
}
}
used[found_idx] = true;
}
false
}
@@ -635,7 +668,9 @@ pub struct HdHomerunAppState {
#[cfg(test)]
mod tests {
use super::{should_use_manual_redirect_for_proxy, should_use_manual_redirects_for_env_vars};
use super::{schedules_changed, should_use_manual_redirect_for_proxy, should_use_manual_redirects_for_env_vars};
use crate::model::ScheduleConfig;
use shared::model::ScheduleTaskType;
#[test]
fn should_use_manual_redirect_for_proxy_only_http_or_https() {
@@ -672,4 +707,48 @@ mod tests {
"http://localhost".to_string(),
)]));
}
#[test]
fn schedules_changed_detects_task_type_changes() {
let a = vec![ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::PlaylistUpdate,
targets: None,
}];
let b = vec![ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::GeoIpUpdate,
targets: None,
}];
assert!(schedules_changed(&a, &b));
}
#[test]
fn schedules_changed_treats_same_entries_as_unchanged() {
let a = vec![
ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::GeoIpUpdate,
targets: None,
},
ScheduleConfig {
schedule: "0 0 8 * * * *".to_string(),
task_type: ScheduleTaskType::PlaylistUpdate,
targets: Some(vec!["a".to_string(), "b".to_string()]),
},
];
let b = vec![
ScheduleConfig {
schedule: "0 0 8 * * * *".to_string(),
task_type: ScheduleTaskType::PlaylistUpdate,
targets: Some(vec!["b".to_string(), "a".to_string()]),
},
ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::GeoIpUpdate,
targets: None,
},
];
assert!(!schedules_changed(&a, &b));
}
}
backend/src/api/model/metadata_update_manager.rs
+89 -44
View File
@@ -596,6 +596,7 @@ impl MetadataUpdateManager {
/// * `task` - The task to process
#[allow(clippy::too_many_lines)]
pub async fn queue_task(&self, input_name: Arc<str>, task: UpdateTask) {
debug!("[Task] Queuing task for input {input_name}: {task}");
// Read app state once and reuse for worker creation when needed.
let app_state_weak = {
let guard = self.app_state.lock().await;
@@ -747,8 +748,12 @@ impl MetadataUpdateManager {
}
} else {
let mut existing = entry.task.lock();
let before = format!("{existing}");
if Self::merge_task_payload(&mut existing, task_to_submit) {
entry.generation.fetch_add(1, Ordering::Relaxed);
debug!("[Task] Merged task for input {input_name}: before={before}, after={existing}");
} else {
debug!("[Task] Task already pending for input {input_name} (no merge needed): {existing}");
}
return SubmitTaskResult::QueuedOrMerged;
}
@@ -987,8 +992,8 @@ impl InputWorker {
let mut task_for_execution = current_task.clone();
if Self::is_resolve_task(&task_for_execution) && self.resolve_exhausted.contains_key(&current_key) {
debug_if_enabled!(
"Skipping exhausted resolve task for input {}: {} (reset window: {}s)",
debug!(
"[Metadata-Task] Skipping task (resolve exhausted) for input {}: {} (reset window: {}s)",
input_name,
task_for_execution,
runtime_settings.resolve_exhaustion_reset_gap_secs
@@ -1003,10 +1008,8 @@ impl InputWorker {
if !skip_execution
&& self.should_skip_recent_no_change_task(&current_key, &task_for_execution, &runtime_settings)
{
debug_if_enabled!(
"Skipping recently-resolved no-change task for input {}: {}",
input_name,
task_for_execution
debug!(
"[Metadata-Task] Skipping task (recently completed with no changes) for input {input_name}: {task_for_execution}",
);
remove_current_task = true;
skip_execution = true;
@@ -1019,19 +1022,21 @@ impl InputWorker {
if let Some(cooldown_until_ts) = tmdb_state.cooldown_until_ts {
if now_ts < cooldown_until_ts {
if let Some(stripped_task) = Self::strip_tmdb_reasons(&task_for_execution) {
debug_if_enabled!(
"TMDB cooldown active for input {}: {} (cooldown_until={}), continuing with non-TMDB reasons",
debug!(
"[Task] TMDB cooldown active for input {}: {}, continuing with non-TMDB reasons (cooldown_until={}, remaining={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
cooldown_until_ts.saturating_sub(now_ts)
);
task_for_execution = stripped_task;
} else {
debug_if_enabled!(
"Skipping TMDB-only task in cooldown for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] Skipping task (TMDB-only in cooldown) for input {}: {} (cooldown_until={}, remaining={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
cooldown_until_ts.saturating_sub(now_ts)
);
self.scheduled_requeues.remove(&current_key);
remove_current_task = true;
@@ -1053,11 +1058,12 @@ impl InputWorker {
if active_retry_domain == RetryDomain::Probe {
if let Some(cooldown_until_ts) = active_state.cooldown_until_ts {
if now_ts < cooldown_until_ts {
debug_if_enabled!(
"Skipping probe task in cooldown for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] Skipping task (probe cooldown) for input {}: {} (cooldown_until={}, remaining={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
cooldown_until_ts.saturating_sub(now_ts)
);
self.scheduled_requeues.remove(&current_key);
remove_current_task = true;
@@ -1069,6 +1075,14 @@ impl InputWorker {
}
if !skip_execution && active_state.next_allowed_at_ts > now_ts {
debug!(
"[Task] Deferring task (retry backoff) for input {}: {} (next_allowed_at={}, wait={}s, attempts={})",
input_name,
task_for_execution,
active_state.next_allowed_at_ts,
active_state.next_allowed_at_ts.saturating_sub(now_ts),
active_state.attempts
);
schedule_requeue_at_ts = Some(active_state.next_allowed_at_ts);
skip_execution = true;
}
@@ -1103,6 +1117,12 @@ impl InputWorker {
}
if !skip_execution {
debug!(
"[Task] Executing task for input {}: {} (retry_domain={:?})",
input_name,
task_for_execution,
Self::retry_domain_for_task(&task_for_execution)
);
let task_result = {
let Some(_pause_guard) = self.wait_for_update_pause_window().await else {
break;
@@ -1129,7 +1149,7 @@ impl InputWorker {
Self::should_trigger_playlist_update_for_task(&task_for_execution, task_outcome.task_changed);
cycle_had_changes |= trigger_playlist_update;
debug!(
"Processed metadata task for input {input_name}: {task_for_execution} (changed={}, trigger_playlist_update={}, tmdb_pending={})",
"[Metadata-Task] Task succeeded for input {input_name}: {task_for_execution} (changed={}, trigger_playlist_update={}, tmdb_pending={})",
task_outcome.task_changed,
trigger_playlist_update,
task_outcome.tmdb_pending
@@ -1151,11 +1171,12 @@ impl InputWorker {
tmdb_state.cooldown_until_ts = Some(cooldown_until_ts);
tmdb_state.last_error =
Some("TMDB lookup completed without matching result".to_string());
debug_if_enabled!(
"TMDB resolve produced no match for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] TMDB resolve produced no match (existing retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
runtime_settings.tmdb_cooldown_secs
);
} else {
state_bundle.clear_domain(RetryDomain::Tmdb);
@@ -1183,11 +1204,12 @@ impl InputWorker {
};
self.retry_states.insert(current_key.clone(), state_bundle.clone());
state_after_success = Some(state_bundle);
debug_if_enabled!(
"TMDB resolve produced no match for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] TMDB resolve produced no match (new retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
runtime_settings.tmdb_cooldown_secs
);
}
@@ -1207,6 +1229,10 @@ impl InputWorker {
&& !task_outcome.tmdb_pending
{
let reasons = Self::task_reason(&task_for_execution);
debug!(
"[Task] Caching no-change result for input {}: {} (reasons={}, ttl={}s)",
input_name, task_for_execution, reasons, runtime_settings.no_change_cache_ttl_secs
);
self.recently_completed_no_change.insert(current_key.clone(), (Instant::now(), reasons));
} else {
self.recently_completed_no_change.remove(&current_key);
@@ -1237,6 +1263,10 @@ impl InputWorker {
}
Err(e) => {
if Self::is_permanent_not_found_error(&e.message) {
debug!(
"[Task] Task failed with permanent not-found for input {}: {} (error={})",
input_name, task_for_execution, e.message
);
let retry_domain = Self::retry_domain_for_task(&task_for_execution);
self.scheduled_requeues.remove(&current_key);
if retry_domain == RetryDomain::Probe {
@@ -1253,11 +1283,12 @@ impl InputWorker {
};
metadata_persist_state = Some(Some(state_bundle_after_update));
remove_current_task = true;
debug_if_enabled!(
"Probe task entered cooldown after permanent not-found for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] Probe task entering cooldown after permanent not-found for input {}: {} (cooldown_until={}, cooldown_duration={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
runtime_settings.probe_cooldown_secs
);
} else {
self.resolve_exhausted.insert(current_key.clone(), now_ts);
@@ -1277,10 +1308,11 @@ impl InputWorker {
}
metadata_persist_state = Some(state_after_clear);
remove_current_task = true;
debug_if_enabled!(
"Resolve task marked exhausted after permanent not-found for input {}: {}",
debug!(
"[Metadata-Task] Resolve task marked exhausted after permanent not-found for input {}: {} (error={})",
input_name,
task_for_execution
task_for_execution,
e.message
);
}
} else if Self::is_transient_worker_error(&e.message) {
@@ -1293,8 +1325,8 @@ impl InputWorker {
Self::compute_retry_delay_secs(current_task.delay(), &runtime_settings);
let retry_delay_i64 = i64::try_from(retry_delay_secs).unwrap_or(i64::MAX);
schedule_requeue_at_ts = Some(now_ts.saturating_add(retry_delay_i64));
debug_if_enabled!(
"Transient task deferral for input {}: {} (retry_in={}s, err={})",
debug!(
"[Metadata-Task] Task deferred (transient error) for input {}: {} (retry_in={}s, error={})",
input_name,
task_for_execution,
retry_delay_secs,
@@ -1350,12 +1382,14 @@ impl InputWorker {
let cooldown_until = state_after_update
.cooldown_until_ts
.map_or_else(|| "none".to_string(), |ts| ts.to_string());
debug_if_enabled!(
"Probe task exhausted for input {}: {} (attempts={}, cooldown_until={})",
debug!(
"Metadata-[Task] Probe task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, cooldown_until={}, error={})",
input_name,
task_for_execution,
state_after_update.attempts,
cooldown_until
max_attempts,
cooldown_until,
e.message
);
} else {
self.resolve_exhausted.insert(current_key.clone(), now_ts);
@@ -1375,22 +1409,26 @@ impl InputWorker {
}
metadata_persist_state = Some(state_after_clear);
remove_current_task = true;
debug_if_enabled!(
"Resolve task exhausted for input {}: {} (attempts={})",
debug!(
"[Metadata-Task] Resolve task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, error={})",
input_name,
task_for_execution,
attempts
attempts,
max_attempts,
e.message
);
}
} else {
schedule_requeue_at_ts = Some(state_after_update.next_allowed_at_ts);
metadata_persist_state = Some(Some(state_bundle_after_update));
debug_if_enabled!(
"Task failed for input {}, scheduling retry: {} (attempt={}, next_allowed_at={}, err={})",
debug!(
"[Metadata-Task] Task failed, scheduling retry for input {}: {} (attempt={}/{}, next_allowed_at={}, backoff={}s, error={})",
input_name,
task_for_execution,
attempts,
max_attempts,
state_after_update.next_allowed_at_ts,
state_after_update.next_allowed_at_ts.saturating_sub(now_ts),
e.message
);
}
@@ -1413,6 +1451,13 @@ impl InputWorker {
}
if let Some(retry_at_ts) = schedule_requeue_at_ts {
debug!(
"[Task] Scheduling requeue for input {}: {:?} (retry_at_ts={}, in={}s)",
input_name,
current_key,
retry_at_ts,
retry_at_ts.saturating_sub(chrono::Utc::now().timestamp())
);
self.schedule_requeue_at(current_key.clone(), retry_at_ts);
}
@@ -1510,8 +1555,8 @@ impl InputWorker {
return;
};
let working_dir = app_state.app_config.config.load().working_dir.clone();
let Ok(storage_path) = get_input_storage_path(input_name, &working_dir).await else {
let storage_dir = app_state.app_config.config.load().storage_dir.clone();
let Ok(storage_path) = get_input_storage_path(input_name, &storage_dir).await else {
warn!("Could not resolve storage path for metadata retry state on input {input_name}");
self.metadata_retry_load_retry_at_ts =
Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs));
@@ -2026,7 +2071,7 @@ impl InputWorker {
let Some(app_state) = app_state_weak.and_then(Weak::upgrade) else { return };
let app_config = &app_state.app_config;
let working_dir = &app_config.config.load().working_dir;
let cfg = app_config.config.load();
let vod_updates = batch_buffer.take_vod_updates();
let series_updates = batch_buffer.take_series_updates();
let live_updates = batch_buffer.take_live_updates();
@@ -2035,7 +2080,7 @@ impl InputWorker {
return;
}
if let Ok(storage_path) = get_input_storage_path(input_name, working_dir).await {
if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await {
if !vod_updates.is_empty() {
let mut updates: Vec<(u32, VideoStreamProperties)> = Vec::with_capacity(vod_updates.len());
for (id, props) in &vod_updates {
@@ -2531,8 +2576,8 @@ impl InputWorker {
}
if let std::collections::hash_map::Entry::Vacant(entry) = db_handles.entry(cluster) {
let working_dir = &app_state.app_config.config.load().working_dir;
if let Ok(storage_path) = get_input_storage_path(input_name, working_dir).await {
let cfg = app_state.app_config.config.load();
if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await {
let file_path = xtream_get_file_path(&storage_path, cluster);
if file_path.exists() {
let lock = app_state.app_config.file_locks.read_lock(&file_path).await;
backend/src/api/model/provider_dns_manager.rs
+37 -133
View File
@@ -1,15 +1,15 @@
use crate::api::model::AppState;
use crate::model::ConfigProvider;
use crate::utils::read_sources_file_from_path;
use crate::repository::{
load_persisted_dns_resolved, next_dns_writer_generation, prune_persisted_dns_resolved_to_runtime,
queue_provider_resolved_snapshot, spawn_dns_resolved_writer, DnsResolvedWriteTx,
};
use log::{debug, warn};
use shared::model::{DnsPrefer, OnResolveErrorPolicy, SourcesConfigDto};
use shared::model::{DnsPrefer, OnResolveErrorPolicy};
use std::collections::HashSet;
use std::io;
use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::fs;
use tokio::net::lookup_host;
use tokio_util::sync::CancellationToken;
@@ -105,129 +105,12 @@ async fn resolve_provider(provider: &Arc<ConfigProvider>) -> ProviderResolveStat
stats
}
fn serialize_sources_for_persist(sources: &SourcesConfigDto) -> io::Result<String> {
let mut serialized = String::new();
let options = serde_saphyr::SerializerOptions {
prefer_block_scalars: false,
..Default::default()
};
serde_saphyr::to_fmt_writer_with_options(&mut serialized, sources, options)
.map_err(|err| io::Error::other(format!("Could not serialize source.yml: {err}")))?;
Ok(serialized)
}
async fn write_sources_file_force(path: &Path, sources: &SourcesConfigDto) -> io::Result<()> {
let serialized = serialize_sources_for_persist(sources)?;
let parent_dir = path.parent().ok_or_else(|| {
io::Error::other(format!(
"Could not write source.yml '{}': missing parent directory",
path.display()
))
})?;
let dest_file_name = path.file_name().and_then(|s| s.to_str()).unwrap_or("source.yml");
let mut tmp_path = parent_dir.to_path_buf();
tmp_path.push(format!(
".{dest_file_name}.tmp-{}-{}",
std::process::id(),
chrono::Local::now().timestamp_nanos_opt().unwrap_or_default()
));
fs::write(&tmp_path, serialized).await?;
match fs::rename(&tmp_path, path).await {
Ok(()) => Ok(()),
Err(err) => {
#[cfg(windows)]
{
// Try to rename again after removing destination (Windows often needs this)
if let Ok(()) = fs::remove_file(path).await {
if fs::rename(&tmp_path, path).await.is_ok() {
return Ok(());
}
// Rename still failed after removing dest - fall through to clean up
}
}
let _ = fs::remove_file(&tmp_path).await;
Err(io::Error::other(format!(
"Could not replace '{}' with '{}': {err}",
path.display(),
tmp_path.display()
)))
}
}
}
async fn persist_provider_resolved_to_source_file(app_state: &Arc<AppState>, provider: &Arc<ConfigProvider>) {
let source_file = {
let paths = app_state.app_config.paths.load();
paths.sources_file_path.clone()
};
let source_path = PathBuf::from(&source_file);
let _lock = app_state.app_config.file_locks.write_lock(&source_path).await;
let mut sources_dto = match read_sources_file_from_path(&source_path, false, false, None).await {
Ok(dto) => dto,
Err(err) => {
warn!(
"Provider dns '{}' failed to read source.yml '{}': {err}",
provider.name,
source_path.display()
);
return;
}
};
let Some(provider_dtos) = sources_dto.provider.as_mut() else {
debug!(
"Provider dns '{}' source.yml '{}' has no provider section to persist resolved values",
provider.name,
source_path.display()
);
return;
};
let Some(provider_dto) = provider_dtos.iter_mut().find(|dto| dto.name.as_ref() == provider.name.as_ref()) else {
warn!(
"Provider dns '{}' not found in source.yml '{}', cannot persist resolved values",
provider.name,
source_path.display()
);
return;
};
let Some(dns_dto) = provider_dto.dns.as_mut() else {
debug!(
"Provider dns '{}' has no dns section in source.yml '{}', skipping resolved persist",
provider.name,
source_path.display()
);
return;
};
let resolved_hosts = {
let snapshot = provider.snapshot_resolved_ordered();
dns_dto.resolved = (!snapshot.is_empty()).then_some(snapshot);
dns_dto.resolved.as_ref().map_or(0, indexmap::IndexMap::len)
};
match write_sources_file_force(&source_path, &sources_dto).await {
Ok(()) => {
debug!(
"Provider dns '{}' persisted dns.resolved to '{}' (hosts={resolved_hosts})",
provider.name,
source_path.display()
);
}
Err(err) => {
warn!(
"Provider dns '{}' failed to persist dns.resolved to '{}': {err}",
provider.name,
source_path.display()
);
}
}
}
fn spawn_provider_dns_task(app_state: Arc<AppState>, provider_name: Arc<str>, cancel: CancellationToken) {
fn spawn_provider_dns_task(
app_state: Arc<AppState>,
provider_name: Arc<str>,
cancel: CancellationToken,
writer_tx: DnsResolvedWriteTx,
) {
tokio::spawn(async move {
let mut refresh_secs = 300_u64;
{
@@ -260,7 +143,7 @@ fn spawn_provider_dns_task(app_state: Arc<AppState>, provider_name: Arc<str>, ca
};
refresh_secs = provider.get_dns_config().map_or(300, |dns| dns.refresh_secs.max(10));
let stats = resolve_provider(&provider).await;
persist_provider_resolved_to_source_file(&app_state, &provider).await;
queue_provider_resolved_snapshot(&writer_tx, &provider).await;
let cache_hosts = provider.snapshot_resolved().len();
debug!(
"Provider dns tick '{}' finished: total_hosts={} resolved={} overridden={} empty={} failed={} cache_hosts={} elapsed_ms={}",
@@ -282,6 +165,7 @@ fn spawn_provider_dns_task(app_state: Arc<AppState>, provider_name: Arc<str>, ca
}
pub fn exec_provider_dns(app_state: &Arc<AppState>, cancel: &CancellationToken) {
let generation = next_dns_writer_generation();
let sources = app_state.app_config.sources.load();
let provider_names: Vec<_> = sources
.provider
@@ -292,13 +176,33 @@ pub fn exec_provider_dns(app_state: &Arc<AppState>, cancel: &CancellationToken)
drop(sources);
if provider_names.is_empty() {
let app_for_prune = Arc::clone(app_state);
tokio::spawn(async move {
prune_persisted_dns_resolved_to_runtime(&app_for_prune).await;
});
debug!("Provider dns manager: no enabled providers found");
return;
}
debug!("Provider dns manager: starting {} provider task(s)", provider_names.len());
let app_for_start = Arc::clone(app_state);
let cancel_for_start = cancel.clone();
tokio::spawn(async move {
// Seed runtime caches first, then start writer/tasks to avoid startup races.
load_persisted_dns_resolved(&app_for_start).await;
if cancel_for_start.is_cancelled() {
return;
}
for provider_name in provider_names {
spawn_provider_dns_task(Arc::clone(app_state), provider_name, cancel.clone());
}
let writer_tx = spawn_dns_resolved_writer(Arc::clone(&app_for_start), cancel_for_start.clone(), generation);
debug!("Provider dns manager: starting {} provider task(s)", provider_names.len());
for provider_name in provider_names {
spawn_provider_dns_task(
Arc::clone(&app_for_start),
provider_name,
cancel_for_start.clone(),
writer_tx.clone(),
);
}
});
}
backend/src/api/model/streams/provider_stream_factory.rs
+2 -1
View File
@@ -25,7 +25,6 @@ use reqwest::{
};
use shared::{
create_bitset,
model::{PlaylistItemType, DEFAULT_USER_AGENT},
utils::{filter_request_header, sanitize_sensitive_info},
};
use std::{
@@ -38,6 +37,8 @@ use std::{
time::{Duration, Instant},
};
use url::Url;
use shared::model::PlaylistItemType;
use shared::utils::DEFAULT_USER_AGENT;
const RETRY_SECONDS: u64 = 5;
const ERR_MAX_RETRY_COUNT: u32 = 5;
backend/src/api/scheduler.rs
+38 -2
View File
@@ -1,6 +1,10 @@
use crate::{
api::{library_scan::{spawn_library_scan, LibraryScanTaskOptions}, model::AppState},
api::{
library_scan::{spawn_library_scan, LibraryScanTaskOptions},
model::AppState,
},
model::{AppConfig, ProcessTargets, ScheduleConfig},
processing::geoip::{update_geoip_db, GeoIpUpdateError},
processing::processor::exec_processing,
utils::exit,
};
@@ -39,6 +43,24 @@ pub fn exec_scheduler(
let schedules: Vec<ScheduleConfig> =
if let Some(schedules) = &config.schedules { schedules.clone() } else { vec![] };
for schedule in schedules {
let task_enabled = match schedule.task_type {
ScheduleTaskType::PlaylistUpdate => true,
ScheduleTaskType::LibraryScan => config.library.as_ref().is_some_and(|library| library.enabled),
ScheduleTaskType::GeoIpUpdate => config
.reverse_proxy
.as_ref()
.and_then(|reverse_proxy| reverse_proxy.geoip.as_ref())
.is_some_and(|geoip| geoip.enabled),
};
if !task_enabled {
log::info!(
"Skipping disabled scheduled task {:?} ({})",
schedule.task_type,
schedule.schedule
);
continue;
}
let expression = schedule.schedule.clone();
let task_type = schedule.task_type;
let exec_targets = get_process_targets(cfg, targets, schedule.targets.as_ref());
@@ -75,6 +97,9 @@ async fn start_scheduler(
ScheduleTaskType::LibraryScan => {
run_library_scan(&client, &app_state);
}
ScheduleTaskType::GeoIpUpdate => {
run_geoip_update(&app_state);
}
}
}
() = cancel.cancelled() => {
@@ -131,7 +156,7 @@ fn run_library_scan(client: &reqwest::Client, app_state: &Arc<AppState>) {
LibraryScanTaskOptions {
force_rescan: false,
message_prefix: "Scheduled ",
working_dir: config.working_dir.clone(),
storage_dir: config.storage_dir.clone(),
},
permit,
);
@@ -140,6 +165,17 @@ fn run_library_scan(client: &reqwest::Client, app_state: &Arc<AppState>) {
}
}
fn run_geoip_update(app_state: &Arc<AppState>) {
let app_state = Arc::clone(app_state);
tokio::spawn(async move {
if let Err(err) = update_geoip_db(&app_state).await {
if !matches!(err, GeoIpUpdateError::Disabled) {
log::error!("Scheduled GeoIp update failed: {err}");
}
}
});
}
pub fn get_process_targets(
cfg: &Arc<AppConfig>,
process_targets: &Arc<ProcessTargets>,
backend/src/api/setup_api.rs
+45 -217
View File
@@ -3,8 +3,8 @@ use crate::{
auth::generate_password_from_input,
model::validate_library_paths_from_dto,
utils::{
file_exists, get_default_path, get_default_web_root_path, read_api_proxy_file, read_config_file,
read_sources_file, read_templates_file, resolve_template_persist_file_path, sanitize_sources_for_persist,
file_exists, get_default_web_root_path_for_home, read_api_proxy_file, read_config_file, read_sources_file,
read_templates_file, resolve_template_persist_file_path, sanitize_sources_for_persist,
},
};
use axum::{
@@ -26,10 +26,10 @@ use shared::{
ApiProxyConfigDto, ApiProxyServerInfoDto, AppConfigDto, ConfigApiDto, ConfigDto, ConfigPaths, PatternTemplate,
SourcesConfigDto, TemplateDefinitionDto, TokenResponse, WebAuthConfigDto, WebUiConfigDto, TOKEN_NO_AUTH,
},
utils::{default_kick_secs, hex_encode, DEFAULT_PORT, DEFAULT_WORKING_DIR, USER_FILE},
utils::{default_kick_secs, hex_encode, DEFAULT_PORT, DEFAULT_STORAGE_DIR, USER_FILE},
};
use std::{
collections::{HashMap, HashSet},
collections::HashSet,
io::ErrorKind,
net::{SocketAddr, UdpSocket},
path::{Component, Path as FsPath, PathBuf},
@@ -40,10 +40,9 @@ use std::{
};
use tokio::sync::{oneshot, Mutex, RwLock};
use tower_http::services::ServeDir;
use shared::utils::DEFAULT_CUSTOM_STREAM_RESPONSE_PATH;
const DEFAULT_SETUP_HOST: &str = "0.0.0.0";
const DEFAULT_SETUP_CUSTOM_STREAM_RESPONSE_PATH: &str = "./resources";
const SETUP_REDACTED_SECRET_VALUE: &str = "__TULIPROX_SETUP_REDACTED__";
#[derive(Clone, Serialize, Deserialize)]
pub struct SetupWebUserCredentialDto {
@@ -69,6 +68,7 @@ pub struct SetupCompleteRequestDto {
struct SetupModeState {
draft: Arc<RwLock<AppConfigDto>>,
home_path: String,
output_dir: PathBuf,
config_file_path: PathBuf,
source_file_path: PathBuf,
@@ -166,7 +166,7 @@ fn create_default_api_proxy_server() -> ApiProxyServerInfoDto {
}
}
fn create_default_config_dto() -> ConfigDto {
fn create_default_config_dto(home_path: &str) -> ConfigDto {
let auth = WebAuthConfigDto {
issuer: "tuliprox".to_string(),
secret: generate_web_auth_secret(),
@@ -180,18 +180,20 @@ fn create_default_config_dto() -> ConfigDto {
api: ConfigApiDto {
host: DEFAULT_SETUP_HOST.to_string(),
port: DEFAULT_PORT,
web_root: get_default_web_root_path().display().to_string(),
web_root: get_default_web_root_path_for_home(FsPath::new(home_path))
.display()
.to_string(),
},
working_dir: get_default_path(DEFAULT_WORKING_DIR).display().to_string(),
custom_stream_response_path: Some(DEFAULT_SETUP_CUSTOM_STREAM_RESPONSE_PATH.to_string()),
storage_dir: Some(DEFAULT_STORAGE_DIR.to_string()),
custom_stream_response_path: Some(DEFAULT_CUSTOM_STREAM_RESPONSE_PATH.to_string()),
web_ui: Some(web_ui),
..ConfigDto::default()
}
}
fn create_default_draft() -> AppConfigDto {
fn create_default_draft(home_path: &str) -> AppConfigDto {
AppConfigDto {
config: create_default_config_dto(),
config: create_default_config_dto(home_path),
sources: SourcesConfigDto::default(),
mappings: None,
templates: None,
@@ -204,7 +206,7 @@ fn create_default_draft() -> AppConfigDto {
}
async fn build_initial_draft(paths: &ConfigPaths) -> AppConfigDto {
let mut draft = create_default_draft();
let mut draft = create_default_draft(paths.home_path.as_str());
if file_exists(&paths.config_file_path) {
match read_config_file(paths.config_file_path.as_str(), true, false) {
@@ -251,7 +253,7 @@ fn generate_web_auth_secret() -> String {
hex_encode(&secret).to_lowercase()
}
fn ensure_setup_defaults(config: &mut ConfigDto) {
fn ensure_setup_defaults(config: &mut ConfigDto, home_path: &str) {
if config.api.host.trim().is_empty() {
config.api.host = DEFAULT_SETUP_HOST.to_string();
}
@@ -259,13 +261,15 @@ fn ensure_setup_defaults(config: &mut ConfigDto) {
config.api.port = DEFAULT_PORT;
}
if config.api.web_root.trim().is_empty() {
config.api.web_root = get_default_web_root_path().display().to_string();
config.api.web_root = get_default_web_root_path_for_home(FsPath::new(home_path))
.display()
.to_string();
}
if config.working_dir.trim().is_empty() {
config.working_dir = get_default_path(DEFAULT_WORKING_DIR).display().to_string();
if config.storage_dir.as_ref().is_none_or(|dir| dir.trim().is_empty()) {
config.storage_dir = Some(DEFAULT_STORAGE_DIR.to_string());
}
if config.custom_stream_response_path.as_ref().is_none_or(|path| path.trim().is_empty()) {
config.custom_stream_response_path = Some(DEFAULT_SETUP_CUSTOM_STREAM_RESPONSE_PATH.to_string());
config.custom_stream_response_path = Some(DEFAULT_CUSTOM_STREAM_RESPONSE_PATH.to_string());
}
if config.web_ui.is_none() {
@@ -288,7 +292,7 @@ fn ensure_setup_defaults(config: &mut ConfigDto) {
}
}
fn setup_bind_values(draft: &AppConfigDto) -> (String, u16, PathBuf) {
fn setup_bind_values(draft: &AppConfigDto, home_path: &str) -> (String, u16, PathBuf) {
let host = if draft.config.api.host.trim().is_empty() {
DEFAULT_SETUP_HOST.to_string()
} else {
@@ -296,18 +300,18 @@ fn setup_bind_values(draft: &AppConfigDto) -> (String, u16, PathBuf) {
};
let port = if draft.config.api.port == 0 { DEFAULT_PORT } else { draft.config.api.port };
let web_root = if draft.config.api.web_root.trim().is_empty() {
get_default_web_root_path()
get_default_web_root_path_for_home(FsPath::new(home_path))
} else {
PathBuf::from(&draft.config.api.web_root)
};
(host, port, web_root)
}
fn resolve_setup_web_dir(web_root: &FsPath) -> Option<PathBuf> {
fn resolve_setup_web_dir(web_root: &FsPath, home_path: &str) -> Option<PathBuf> {
if web_root.exists() && web_root.is_dir() {
return Some(web_root.to_path_buf());
}
let fallback = get_default_web_root_path();
let fallback = get_default_web_root_path_for_home(FsPath::new(home_path));
if fallback.exists() && fallback.is_dir() {
return Some(fallback);
}
@@ -322,126 +326,6 @@ fn api_proxy_or_default(draft: &AppConfigDto) -> ApiProxyConfigDto {
})
}
fn is_setup_redacted_value(value: &str) -> bool { value == SETUP_REDACTED_SECRET_VALUE }
fn redact_non_empty_secret(value: &mut String) {
if !value.trim().is_empty() {
*value = SETUP_REDACTED_SECRET_VALUE.to_string();
}
}
fn redact_optional_secret(value: &mut Option<String>) {
if value.as_ref().is_some_and(|entry| !entry.trim().is_empty()) {
*value = Some(SETUP_REDACTED_SECRET_VALUE.to_string());
}
}
fn redact_api_proxy_user_credentials(api_proxy: &mut ApiProxyConfigDto) {
for target in &mut api_proxy.user {
for user in &mut target.credentials {
redact_non_empty_secret(&mut user.password);
redact_optional_secret(&mut user.token);
}
}
}
fn redact_app_config_for_setup(mut app_config: AppConfigDto) -> AppConfigDto {
if let Some(web_ui) = app_config.config.web_ui.as_mut() {
if let Some(auth) = web_ui.auth.as_mut() {
redact_non_empty_secret(&mut auth.secret);
}
}
if let Some(api_proxy) = app_config.api_proxy.as_mut() {
redact_api_proxy_user_credentials(api_proxy);
}
app_config
}
fn redact_api_proxy_for_setup(mut api_proxy: ApiProxyConfigDto) -> ApiProxyConfigDto {
redact_api_proxy_user_credentials(&mut api_proxy);
api_proxy
}
fn restore_redacted_web_auth_secret(app_config: &mut AppConfigDto, draft: &AppConfigDto) {
let Some(web_ui) = app_config.config.web_ui.as_mut() else {
return;
};
let Some(auth) = web_ui.auth.as_mut() else {
return;
};
if !is_setup_redacted_value(&auth.secret) {
return;
}
let draft_secret = draft
.config
.web_ui
.as_ref()
.and_then(|draft_web_ui| draft_web_ui.auth.as_ref())
.map(|draft_auth| draft_auth.secret.trim().to_string())
.filter(|secret| !secret.is_empty());
if let Some(draft_secret) = draft_secret {
auth.secret = draft_secret;
}
}
fn restore_redacted_api_proxy_credentials(
api_proxy: &mut ApiProxyConfigDto,
draft_api_proxy: Option<&ApiProxyConfigDto>,
) {
let mut credentials_by_username: HashMap<String, (String, Option<String>)> = HashMap::new();
if let Some(draft) = draft_api_proxy {
for target in &draft.user {
for user in &target.credentials {
credentials_by_username.insert(user.username.clone(), (user.password.clone(), user.token.clone()));
}
}
}
for target in &mut api_proxy.user {
for user in &mut target.credentials {
let draft_credentials = credentials_by_username.get(&user.username);
if is_setup_redacted_value(&user.password) {
if let Some((password, _)) = draft_credentials {
user.password = password.clone();
}
}
if user.token.as_deref().is_some_and(is_setup_redacted_value) {
if let Some((_, token)) = draft_credentials {
user.token = token.clone();
}
}
}
}
}
fn restore_redacted_setup_values(app_config: &mut AppConfigDto, draft: &AppConfigDto) {
restore_redacted_web_auth_secret(app_config, draft);
if let Some(api_proxy) = app_config.api_proxy.as_mut() {
restore_redacted_api_proxy_credentials(api_proxy, draft.api_proxy.as_ref());
}
}
fn has_unresolved_redacted_setup_values(app_config: &AppConfigDto) -> bool {
if app_config
.config
.web_ui
.as_ref()
.and_then(|web_ui| web_ui.auth.as_ref())
.is_some_and(|auth| is_setup_redacted_value(&auth.secret))
{
return true;
}
app_config.api_proxy.as_ref().is_some_and(|api_proxy| {
api_proxy.user.iter().any(|target| {
target.credentials.iter().any(|user| {
is_setup_redacted_value(&user.password) || user.token.as_deref().is_some_and(is_setup_redacted_value)
})
})
})
}
fn setup_templates_from_request(app_config: &AppConfigDto) -> Option<Vec<PatternTemplate>> {
app_config
.templates
@@ -518,12 +402,12 @@ async fn setup_token_refresh() -> impl IntoResponse + Send {
async fn setup_get_config(State(state): State<Arc<SetupModeState>>) -> impl IntoResponse + Send {
let draft = state.draft.read().await.clone();
axum::Json(redact_app_config_for_setup(draft)).into_response()
axum::Json(draft).into_response()
}
async fn setup_get_api_proxy(State(state): State<Arc<SetupModeState>>) -> impl IntoResponse + Send {
let draft = state.draft.read().await.clone();
axum::Json(redact_api_proxy_for_setup(api_proxy_or_default(&draft))).into_response()
axum::Json(api_proxy_or_default(&draft)).into_response()
}
async fn setup_config_json(State(state): State<Arc<SetupModeState>>) -> impl IntoResponse + Send {
@@ -751,19 +635,7 @@ async fn setup_complete_inner(
state: Arc<SetupModeState>,
mut req: SetupCompleteRequestDto,
) -> axum::response::Response {
let draft_snapshot = state.draft.read().await.clone();
restore_redacted_setup_values(&mut req.app_config, &draft_snapshot);
if has_unresolved_redacted_setup_values(&req.app_config) {
return (
StatusCode::BAD_REQUEST,
axum::Json(json!({
"error": "One or more redacted secret fields could not be restored. Please re-enter secret values before saving setup."
})),
)
.into_response();
}
ensure_setup_defaults(&mut req.app_config.config);
ensure_setup_defaults(&mut req.app_config.config, state.home_path.as_str());
if let Err(err) = req.app_config.config.prepare(false) {
return (StatusCode::BAD_REQUEST, axum::Json(json!({ "error": err.to_string() }))).into_response();
@@ -983,13 +855,14 @@ fn create_compression_layer() -> tower_http::compression::CompressionLayer {
pub async fn start_setup_server(paths: &ConfigPaths, missing_files: &[String]) -> Result<(), TuliproxError> {
let draft = build_initial_draft(paths).await;
let (host, port, web_root) = setup_bind_values(&draft);
let web_dir = resolve_setup_web_dir(&web_root)
let (host, port, web_root) = setup_bind_values(&draft, paths.home_path.as_str());
let web_dir = resolve_setup_web_dir(&web_root, paths.home_path.as_str())
.ok_or_else(|| info_err!("Setup mode requires a web directory. Tried '{}'", web_root.display(),))?;
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
let state = Arc::new(SetupModeState {
draft: Arc::new(RwLock::new(draft)),
home_path: paths.home_path.clone(),
output_dir: PathBuf::from(&paths.config_path),
config_file_path: PathBuf::from(&paths.config_file_path),
source_file_path: PathBuf::from(&paths.sources_file_path),
@@ -1001,7 +874,7 @@ pub async fn start_setup_server(paths: &ConfigPaths, missing_files: &[String]) -
shutdown_tx: Mutex::new(Some(shutdown_tx)),
});
info!("Setup mode enabled. Missing required config files: {}", missing_files.join(", "));
info!("Setup mode enabled.");
info!("Setup output directory: {}", state.output_dir.display());
info!("Setup web root: {}", state.web_dir.display());
info!("Setup server running: http://{host}:{port}");
@@ -1048,25 +921,12 @@ pub async fn start_setup_server(paths: &ConfigPaths, missing_files: &[String]) -
#[cfg(test)]
mod tests {
use super::{
has_unresolved_redacted_setup_values, redact_app_config_for_setup, restore_redacted_setup_values,
SETUP_REDACTED_SECRET_VALUE,
};
use shared::model::{ApiProxyConfigDto, AppConfigDto, TargetUserDto, WebAuthConfigDto, WebUiConfigDto};
use super::api_proxy_or_default;
use shared::model::{ApiProxyConfigDto, AppConfigDto, TargetUserDto};
fn sample_app_config() -> AppConfigDto {
AppConfigDto {
config: shared::model::ConfigDto {
web_ui: Some(WebUiConfigDto {
auth: Some(WebAuthConfigDto {
issuer: "tuliprox".to_string(),
secret: "very-secret-value".to_string(),
..Default::default()
}),
..Default::default()
}),
..Default::default()
},
#[test]
fn setup_api_proxy_or_default_preserves_existing_credentials() {
let app_config = AppConfigDto {
api_proxy: Some(ApiProxyConfigDto {
user: vec![TargetUserDto {
target: "target-a".to_string(),
@@ -1080,49 +940,17 @@ mod tests {
..Default::default()
}),
..Default::default()
}
}
#[test]
fn setup_redaction_masks_web_auth_and_api_proxy_credentials() {
let redacted = redact_app_config_for_setup(sample_app_config());
let auth_secret =
redacted.config.web_ui.as_ref().and_then(|web_ui| web_ui.auth.as_ref()).map(|auth| auth.secret.as_str());
assert_eq!(auth_secret, Some(SETUP_REDACTED_SECRET_VALUE));
let creds = &redacted.api_proxy.expect("api_proxy should be present").user[0].credentials[0];
assert_eq!(creds.password, SETUP_REDACTED_SECRET_VALUE);
assert_eq!(creds.token.as_deref(), Some(SETUP_REDACTED_SECRET_VALUE));
}
#[test]
fn setup_restore_replaces_redacted_values_from_existing_draft() {
let draft = sample_app_config();
let mut submitted = redact_app_config_for_setup(draft.clone());
restore_redacted_setup_values(&mut submitted, &draft);
let auth_secret =
submitted.config.web_ui.as_ref().and_then(|web_ui| web_ui.auth.as_ref()).map(|auth| auth.secret.as_str());
assert_eq!(auth_secret, Some("very-secret-value"));
let creds = &submitted.api_proxy.as_ref().expect("api_proxy should be present").user[0].credentials[0];
};
let api_proxy = api_proxy_or_default(&app_config);
let creds = &api_proxy.user[0].credentials[0];
assert_eq!(creds.password, "alice-password");
assert_eq!(creds.token.as_deref(), Some("alice-token"));
assert!(!has_unresolved_redacted_setup_values(&submitted));
}
#[test]
fn setup_restore_keeps_unmatched_redacted_credentials_as_unresolved() {
let draft = sample_app_config();
let mut submitted = redact_app_config_for_setup(draft);
if let Some(api_proxy) = submitted.api_proxy.as_mut() {
api_proxy.user[0].credentials[0].username = "bob".to_string();
}
restore_redacted_setup_values(&mut submitted, &AppConfigDto::default());
assert!(has_unresolved_redacted_setup_values(&submitted));
fn setup_api_proxy_or_default_creates_server_when_missing() {
let api_proxy = api_proxy_or_default(&AppConfigDto::default());
assert_eq!(api_proxy.server.len(), 1);
assert!(!api_proxy.server[0].host.trim().is_empty());
}
}