euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-18 13:52:20 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

Feature/staged input and path improvements (#632)

Browse Source 
* No template file not found error message
* Setup wizard improvmenets
* path improvements
* staged input fix
* path improvements
* Provider dns resolved ips now writing to a file instead of source.yml
* Playlist update hard kill after 1h
* matadata update tmdb fix
* Web UI fixes
* GeoIp update schedule
* batch:// prefix  for aliases
This commit is contained in:
euzu
2026-03-06 17:29:42 +01:00
committed by GitHub
parent c5632d0c98
commit 737d04d08b
82 changed files with 2506 additions and 1433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/api/model/active_provider_manager.rs
+2
View File
@@ -708,7 +708,9 @@ mod tests {
api_proxy: Arc::new(ArcSwapOption::default()),
file_locks: Arc::new(FileLockManager::default()),
paths: Arc::new(ArcSwap::from_pointee(ConfigPaths {
home_path: String::new(),
config_path: String::new(),
storage_path: String::new(),
config_file_path: String::new(),
sources_file_path: String::new(),
mapping_file_path: None,
backend/src/api/model/app_state.rs
+96 -17
View File
@@ -266,7 +266,7 @@ fn build_http_client_with_fallback(
pub fn create_cache(config: &Config) -> Option<Arc<Mutex<LRUResourceCache>>> {
let lru_cache = config.reverse_proxy.as_ref().and_then(|r| r.cache.as_ref()).and_then(|c| {
if c.enabled {
Some(LRUResourceCache::new(c.size, c.dir.as_str()))
Some(LRUResourceCache::new(c.size, c.directory.as_str()))
} else {
None
}
@@ -341,20 +341,23 @@ pub struct AppState {
impl AppState {
pub(in crate::api::model) async fn set_config(&self, config: Config) -> Result<UpdateChanges, TuliproxError> {
let old_storage_dir = self.app_config.config.load().storage_dir.clone();
let changes = self.detect_changes_for_config(&config);
config.update_runtime();
let use_geoip = config.is_geoip_enabled();
let working_dir = config.working_dir.clone();
let storage_dir = config.storage_dir.clone();
self.active_users.update_config(&config);
self.app_config.set_config(config)?;
self.active_provider.update_config(&self.app_config).await;
self.update_config().await?;
if changes.flags.contains(UpdateChangesFlags::Geoip) {
let geoip_reload_needed =
changes.flags.contains(UpdateChangesFlags::Geoip) || (use_geoip && old_storage_dir != storage_dir);
if geoip_reload_needed {
let new_geoip = if use_geoip {
let path = get_geoip_path(&working_dir);
let path = get_geoip_path(&storage_dir);
let _file_lock = self.app_config.file_locks.read_lock(&path).await;
GeoIp::load(&path).ok().map(Arc::new)
} else {
@@ -381,7 +384,7 @@ impl AppState {
.reverse_proxy
.as_ref()
.and_then(|r| r.cache.as_ref())
.map_or((false, 0, ""), |c| (c.enabled, c.size, c.dir.as_str()));
.map_or((false, 0, ""), |c| (c.enabled, c.size, c.directory.as_str()));
if let Some(cache) = self.cache.load().as_ref() {
if enabled {
@@ -401,6 +404,22 @@ impl AppState {
sources: SourcesConfig,
) -> Result<UpdateChanges, TuliproxError> {
let changes = self.detect_changes_for_sources(&sources);
// Carry over DNS caches from old providers so resolved IPs survive hot-reloads
// without waiting for the background resolver or the persisted-file seed.
{
let old_sources = self.app_config.sources.load();
for new_provider in &sources.provider {
if let Some(old_provider) = old_sources.get_provider_by_name(&new_provider.name) {
if new_provider.get_dns_config().is_some_and(|cfg| cfg.enabled) {
for (host, ips) in old_provider.snapshot_resolved() {
if !ips.is_empty() && new_provider.dns_cache.ip_count(&host) == 0 {
new_provider.dns_cache.store_resolved(&host, ips);
}
}
}
}
}
}
self.app_config.set_sources(sources)?;
self.active_provider.update_config(&self.app_config).await;
@@ -420,6 +439,9 @@ impl AppState {
let old_config = self.app_config.config.load();
let changed_schedules =
change_detect!(schedules_changed, old_config.schedules.as_ref(), config.schedules.as_ref());
let library_enabled = config.library.as_ref().is_some_and(|library| library.enabled);
let old_library_enabled = old_config.library.as_ref().is_some_and(|library| library.enabled);
let changed_library_enabled = library_enabled != old_library_enabled;
let changed_hdhomerun =
change_detect!(hdhomerun_changed, old_config.hdhomerun.as_ref(), config.hdhomerun.as_ref());
let changed_file_watch =
@@ -430,7 +452,7 @@ impl AppState {
let geoip_enabled_old = old_config.is_geoip_enabled();
let mut changes = UpdateChanges { flags: UpdateChangesFlagsSet::new(), targets: None };
changes.set_flag_if(changed_schedules, UpdateChangesFlags::Scheduler);
changes.set_flag_if(changed_schedules || changed_library_enabled || geoip_enabled != geoip_enabled_old, UpdateChangesFlags::Scheduler);
changes.set_flag_if(changed_hdhomerun, UpdateChangesFlags::Hdhomerun);
changes.set_flag_if(changed_file_watch, UpdateChangesFlags::FileWatch);
changes.set_flag_if(geoip_enabled != geoip_enabled_old, UpdateChangesFlags::Geoip);
@@ -582,19 +604,30 @@ fn schedules_changed(a: &[ScheduleConfig], b: &[ScheduleConfig]) -> bool {
if a.len() != b.len() {
return true;
}
let mut used = vec![false; b.len()];
for schedule in a {
let Some(found) = b.iter().find(|&s| s.schedule == schedule.schedule) else {
let Some(found_idx) = b.iter().enumerate().find_map(|(idx, candidate)| {
if used[idx]
|| candidate.schedule != schedule.schedule
|| candidate.task_type != schedule.task_type
{
return None;
}
let targets_match = match (schedule.targets.as_ref(), candidate.targets.as_ref()) {
(None, None) => true,
(Some(_), None) | (None, Some(_)) => false,
(Some(a_targets), Some(b_targets)) => small_vecs_equal_unordered(a_targets, b_targets),
};
if targets_match {
Some(idx)
} else {
None
}
}) else {
return true;
};
match (schedule.targets.as_ref(), found.targets.as_ref()) {
(None, None) => {}
(Some(_), None) | (None, Some(_)) => return true,
(Some(a_targets), Some(b_targets)) => {
if !small_vecs_equal_unordered(a_targets, b_targets) {
return true;
}
}
}
used[found_idx] = true;
}
false
}
@@ -635,7 +668,9 @@ pub struct HdHomerunAppState {
#[cfg(test)]
mod tests {
use super::{should_use_manual_redirect_for_proxy, should_use_manual_redirects_for_env_vars};
use super::{schedules_changed, should_use_manual_redirect_for_proxy, should_use_manual_redirects_for_env_vars};
use crate::model::ScheduleConfig;
use shared::model::ScheduleTaskType;
#[test]
fn should_use_manual_redirect_for_proxy_only_http_or_https() {
@@ -672,4 +707,48 @@ mod tests {
"http://localhost".to_string(),
)]));
}
#[test]
fn schedules_changed_detects_task_type_changes() {
let a = vec![ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::PlaylistUpdate,
targets: None,
}];
let b = vec![ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::GeoIpUpdate,
targets: None,
}];
assert!(schedules_changed(&a, &b));
}
#[test]
fn schedules_changed_treats_same_entries_as_unchanged() {
let a = vec![
ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::GeoIpUpdate,
targets: None,
},
ScheduleConfig {
schedule: "0 0 8 * * * *".to_string(),
task_type: ScheduleTaskType::PlaylistUpdate,
targets: Some(vec!["a".to_string(), "b".to_string()]),
},
];
let b = vec![
ScheduleConfig {
schedule: "0 0 8 * * * *".to_string(),
task_type: ScheduleTaskType::PlaylistUpdate,
targets: Some(vec!["b".to_string(), "a".to_string()]),
},
ScheduleConfig {
schedule: "0 0 3 * * * *".to_string(),
task_type: ScheduleTaskType::GeoIpUpdate,
targets: None,
},
];
assert!(!schedules_changed(&a, &b));
}
}
backend/src/api/model/metadata_update_manager.rs
+89 -44
View File
@@ -596,6 +596,7 @@ impl MetadataUpdateManager {
/// * `task` - The task to process
#[allow(clippy::too_many_lines)]
pub async fn queue_task(&self, input_name: Arc<str>, task: UpdateTask) {
debug!("[Task] Queuing task for input {input_name}: {task}");
// Read app state once and reuse for worker creation when needed.
let app_state_weak = {
let guard = self.app_state.lock().await;
@@ -747,8 +748,12 @@ impl MetadataUpdateManager {
}
} else {
let mut existing = entry.task.lock();
let before = format!("{existing}");
if Self::merge_task_payload(&mut existing, task_to_submit) {
entry.generation.fetch_add(1, Ordering::Relaxed);
debug!("[Task] Merged task for input {input_name}: before={before}, after={existing}");
} else {
debug!("[Task] Task already pending for input {input_name} (no merge needed): {existing}");
}
return SubmitTaskResult::QueuedOrMerged;
}
@@ -987,8 +992,8 @@ impl InputWorker {
let mut task_for_execution = current_task.clone();
if Self::is_resolve_task(&task_for_execution) && self.resolve_exhausted.contains_key(&current_key) {
debug_if_enabled!(
"Skipping exhausted resolve task for input {}: {} (reset window: {}s)",
debug!(
"[Metadata-Task] Skipping task (resolve exhausted) for input {}: {} (reset window: {}s)",
input_name,
task_for_execution,
runtime_settings.resolve_exhaustion_reset_gap_secs
@@ -1003,10 +1008,8 @@ impl InputWorker {
if !skip_execution
&& self.should_skip_recent_no_change_task(&current_key, &task_for_execution, &runtime_settings)
{
debug_if_enabled!(
"Skipping recently-resolved no-change task for input {}: {}",
input_name,
task_for_execution
debug!(
"[Metadata-Task] Skipping task (recently completed with no changes) for input {input_name}: {task_for_execution}",
);
remove_current_task = true;
skip_execution = true;
@@ -1019,19 +1022,21 @@ impl InputWorker {
if let Some(cooldown_until_ts) = tmdb_state.cooldown_until_ts {
if now_ts < cooldown_until_ts {
if let Some(stripped_task) = Self::strip_tmdb_reasons(&task_for_execution) {
debug_if_enabled!(
"TMDB cooldown active for input {}: {} (cooldown_until={}), continuing with non-TMDB reasons",
debug!(
"[Task] TMDB cooldown active for input {}: {}, continuing with non-TMDB reasons (cooldown_until={}, remaining={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
cooldown_until_ts.saturating_sub(now_ts)
);
task_for_execution = stripped_task;
} else {
debug_if_enabled!(
"Skipping TMDB-only task in cooldown for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] Skipping task (TMDB-only in cooldown) for input {}: {} (cooldown_until={}, remaining={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
cooldown_until_ts.saturating_sub(now_ts)
);
self.scheduled_requeues.remove(&current_key);
remove_current_task = true;
@@ -1053,11 +1058,12 @@ impl InputWorker {
if active_retry_domain == RetryDomain::Probe {
if let Some(cooldown_until_ts) = active_state.cooldown_until_ts {
if now_ts < cooldown_until_ts {
debug_if_enabled!(
"Skipping probe task in cooldown for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] Skipping task (probe cooldown) for input {}: {} (cooldown_until={}, remaining={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
cooldown_until_ts.saturating_sub(now_ts)
);
self.scheduled_requeues.remove(&current_key);
remove_current_task = true;
@@ -1069,6 +1075,14 @@ impl InputWorker {
}
if !skip_execution && active_state.next_allowed_at_ts > now_ts {
debug!(
"[Task] Deferring task (retry backoff) for input {}: {} (next_allowed_at={}, wait={}s, attempts={})",
input_name,
task_for_execution,
active_state.next_allowed_at_ts,
active_state.next_allowed_at_ts.saturating_sub(now_ts),
active_state.attempts
);
schedule_requeue_at_ts = Some(active_state.next_allowed_at_ts);
skip_execution = true;
}
@@ -1103,6 +1117,12 @@ impl InputWorker {
}
if !skip_execution {
debug!(
"[Task] Executing task for input {}: {} (retry_domain={:?})",
input_name,
task_for_execution,
Self::retry_domain_for_task(&task_for_execution)
);
let task_result = {
let Some(_pause_guard) = self.wait_for_update_pause_window().await else {
break;
@@ -1129,7 +1149,7 @@ impl InputWorker {
Self::should_trigger_playlist_update_for_task(&task_for_execution, task_outcome.task_changed);
cycle_had_changes |= trigger_playlist_update;
debug!(
"Processed metadata task for input {input_name}: {task_for_execution} (changed={}, trigger_playlist_update={}, tmdb_pending={})",
"[Metadata-Task] Task succeeded for input {input_name}: {task_for_execution} (changed={}, trigger_playlist_update={}, tmdb_pending={})",
task_outcome.task_changed,
trigger_playlist_update,
task_outcome.tmdb_pending
@@ -1151,11 +1171,12 @@ impl InputWorker {
tmdb_state.cooldown_until_ts = Some(cooldown_until_ts);
tmdb_state.last_error =
Some("TMDB lookup completed without matching result".to_string());
debug_if_enabled!(
"TMDB resolve produced no match for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] TMDB resolve produced no match (existing retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
runtime_settings.tmdb_cooldown_secs
);
} else {
state_bundle.clear_domain(RetryDomain::Tmdb);
@@ -1183,11 +1204,12 @@ impl InputWorker {
};
self.retry_states.insert(current_key.clone(), state_bundle.clone());
state_after_success = Some(state_bundle);
debug_if_enabled!(
"TMDB resolve produced no match for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] TMDB resolve produced no match (new retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
runtime_settings.tmdb_cooldown_secs
);
}
@@ -1207,6 +1229,10 @@ impl InputWorker {
&& !task_outcome.tmdb_pending
{
let reasons = Self::task_reason(&task_for_execution);
debug!(
"[Task] Caching no-change result for input {}: {} (reasons={}, ttl={}s)",
input_name, task_for_execution, reasons, runtime_settings.no_change_cache_ttl_secs
);
self.recently_completed_no_change.insert(current_key.clone(), (Instant::now(), reasons));
} else {
self.recently_completed_no_change.remove(&current_key);
@@ -1237,6 +1263,10 @@ impl InputWorker {
}
Err(e) => {
if Self::is_permanent_not_found_error(&e.message) {
debug!(
"[Task] Task failed with permanent not-found for input {}: {} (error={})",
input_name, task_for_execution, e.message
);
let retry_domain = Self::retry_domain_for_task(&task_for_execution);
self.scheduled_requeues.remove(&current_key);
if retry_domain == RetryDomain::Probe {
@@ -1253,11 +1283,12 @@ impl InputWorker {
};
metadata_persist_state = Some(Some(state_bundle_after_update));
remove_current_task = true;
debug_if_enabled!(
"Probe task entered cooldown after permanent not-found for input {}: {} (cooldown_until={})",
debug!(
"[Metadata-Task] Probe task entering cooldown after permanent not-found for input {}: {} (cooldown_until={}, cooldown_duration={}s)",
input_name,
task_for_execution,
cooldown_until_ts
cooldown_until_ts,
runtime_settings.probe_cooldown_secs
);
} else {
self.resolve_exhausted.insert(current_key.clone(), now_ts);
@@ -1277,10 +1308,11 @@ impl InputWorker {
}
metadata_persist_state = Some(state_after_clear);
remove_current_task = true;
debug_if_enabled!(
"Resolve task marked exhausted after permanent not-found for input {}: {}",
debug!(
"[Metadata-Task] Resolve task marked exhausted after permanent not-found for input {}: {} (error={})",
input_name,
task_for_execution
task_for_execution,
e.message
);
}
} else if Self::is_transient_worker_error(&e.message) {
@@ -1293,8 +1325,8 @@ impl InputWorker {
Self::compute_retry_delay_secs(current_task.delay(), &runtime_settings);
let retry_delay_i64 = i64::try_from(retry_delay_secs).unwrap_or(i64::MAX);
schedule_requeue_at_ts = Some(now_ts.saturating_add(retry_delay_i64));
debug_if_enabled!(
"Transient task deferral for input {}: {} (retry_in={}s, err={})",
debug!(
"[Metadata-Task] Task deferred (transient error) for input {}: {} (retry_in={}s, error={})",
input_name,
task_for_execution,
retry_delay_secs,
@@ -1350,12 +1382,14 @@ impl InputWorker {
let cooldown_until = state_after_update
.cooldown_until_ts
.map_or_else(|| "none".to_string(), |ts| ts.to_string());
debug_if_enabled!(
"Probe task exhausted for input {}: {} (attempts={}, cooldown_until={})",
debug!(
"Metadata-[Task] Probe task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, cooldown_until={}, error={})",
input_name,
task_for_execution,
state_after_update.attempts,
cooldown_until
max_attempts,
cooldown_until,
e.message
);
} else {
self.resolve_exhausted.insert(current_key.clone(), now_ts);
@@ -1375,22 +1409,26 @@ impl InputWorker {
}
metadata_persist_state = Some(state_after_clear);
remove_current_task = true;
debug_if_enabled!(
"Resolve task exhausted for input {}: {} (attempts={})",
debug!(
"[Metadata-Task] Resolve task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, error={})",
input_name,
task_for_execution,
attempts
attempts,
max_attempts,
e.message
);
}
} else {
schedule_requeue_at_ts = Some(state_after_update.next_allowed_at_ts);
metadata_persist_state = Some(Some(state_bundle_after_update));
debug_if_enabled!(
"Task failed for input {}, scheduling retry: {} (attempt={}, next_allowed_at={}, err={})",
debug!(
"[Metadata-Task] Task failed, scheduling retry for input {}: {} (attempt={}/{}, next_allowed_at={}, backoff={}s, error={})",
input_name,
task_for_execution,
attempts,
max_attempts,
state_after_update.next_allowed_at_ts,
state_after_update.next_allowed_at_ts.saturating_sub(now_ts),
e.message
);
}
@@ -1413,6 +1451,13 @@ impl InputWorker {
}
if let Some(retry_at_ts) = schedule_requeue_at_ts {
debug!(
"[Task] Scheduling requeue for input {}: {:?} (retry_at_ts={}, in={}s)",
input_name,
current_key,
retry_at_ts,
retry_at_ts.saturating_sub(chrono::Utc::now().timestamp())
);
self.schedule_requeue_at(current_key.clone(), retry_at_ts);
}
@@ -1510,8 +1555,8 @@ impl InputWorker {
return;
};
let working_dir = app_state.app_config.config.load().working_dir.clone();
let Ok(storage_path) = get_input_storage_path(input_name, &working_dir).await else {
let storage_dir = app_state.app_config.config.load().storage_dir.clone();
let Ok(storage_path) = get_input_storage_path(input_name, &storage_dir).await else {
warn!("Could not resolve storage path for metadata retry state on input {input_name}");
self.metadata_retry_load_retry_at_ts =
Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs));
@@ -2026,7 +2071,7 @@ impl InputWorker {
let Some(app_state) = app_state_weak.and_then(Weak::upgrade) else { return };
let app_config = &app_state.app_config;
let working_dir = &app_config.config.load().working_dir;
let cfg = app_config.config.load();
let vod_updates = batch_buffer.take_vod_updates();
let series_updates = batch_buffer.take_series_updates();
let live_updates = batch_buffer.take_live_updates();
@@ -2035,7 +2080,7 @@ impl InputWorker {
return;
}
if let Ok(storage_path) = get_input_storage_path(input_name, working_dir).await {
if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await {
if !vod_updates.is_empty() {
let mut updates: Vec<(u32, VideoStreamProperties)> = Vec::with_capacity(vod_updates.len());
for (id, props) in &vod_updates {
@@ -2531,8 +2576,8 @@ impl InputWorker {
}
if let std::collections::hash_map::Entry::Vacant(entry) = db_handles.entry(cluster) {
let working_dir = &app_state.app_config.config.load().working_dir;
if let Ok(storage_path) = get_input_storage_path(input_name, working_dir).await {
let cfg = app_state.app_config.config.load();
if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await {
let file_path = xtream_get_file_path(&storage_path, cluster);
if file_path.exists() {
let lock = app_state.app_config.file_locks.read_lock(&file_path).await;
backend/src/api/model/provider_dns_manager.rs
+37 -133
View File
@@ -1,15 +1,15 @@
use crate::api::model::AppState;
use crate::model::ConfigProvider;
use crate::utils::read_sources_file_from_path;
use crate::repository::{
load_persisted_dns_resolved, next_dns_writer_generation, prune_persisted_dns_resolved_to_runtime,
queue_provider_resolved_snapshot, spawn_dns_resolved_writer, DnsResolvedWriteTx,
};
use log::{debug, warn};
use shared::model::{DnsPrefer, OnResolveErrorPolicy, SourcesConfigDto};
use shared::model::{DnsPrefer, OnResolveErrorPolicy};
use std::collections::HashSet;
use std::io;
use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::fs;
use tokio::net::lookup_host;
use tokio_util::sync::CancellationToken;
@@ -105,129 +105,12 @@ async fn resolve_provider(provider: &Arc<ConfigProvider>) -> ProviderResolveStat
stats
}
fn serialize_sources_for_persist(sources: &SourcesConfigDto) -> io::Result<String> {
let mut serialized = String::new();
let options = serde_saphyr::SerializerOptions {
prefer_block_scalars: false,
..Default::default()
};
serde_saphyr::to_fmt_writer_with_options(&mut serialized, sources, options)
.map_err(|err| io::Error::other(format!("Could not serialize source.yml: {err}")))?;
Ok(serialized)
}
async fn write_sources_file_force(path: &Path, sources: &SourcesConfigDto) -> io::Result<()> {
let serialized = serialize_sources_for_persist(sources)?;
let parent_dir = path.parent().ok_or_else(|| {
io::Error::other(format!(
"Could not write source.yml '{}': missing parent directory",
path.display()
))
})?;
let dest_file_name = path.file_name().and_then(|s| s.to_str()).unwrap_or("source.yml");
let mut tmp_path = parent_dir.to_path_buf();
tmp_path.push(format!(
".{dest_file_name}.tmp-{}-{}",
std::process::id(),
chrono::Local::now().timestamp_nanos_opt().unwrap_or_default()
));
fs::write(&tmp_path, serialized).await?;
match fs::rename(&tmp_path, path).await {
Ok(()) => Ok(()),
Err(err) => {
#[cfg(windows)]
{
// Try to rename again after removing destination (Windows often needs this)
if let Ok(()) = fs::remove_file(path).await {
if fs::rename(&tmp_path, path).await.is_ok() {
return Ok(());
}
// Rename still failed after removing dest - fall through to clean up
}
}
let _ = fs::remove_file(&tmp_path).await;
Err(io::Error::other(format!(
"Could not replace '{}' with '{}': {err}",
path.display(),
tmp_path.display()
)))
}
}
}
async fn persist_provider_resolved_to_source_file(app_state: &Arc<AppState>, provider: &Arc<ConfigProvider>) {
let source_file = {
let paths = app_state.app_config.paths.load();
paths.sources_file_path.clone()
};
let source_path = PathBuf::from(&source_file);
let _lock = app_state.app_config.file_locks.write_lock(&source_path).await;
let mut sources_dto = match read_sources_file_from_path(&source_path, false, false, None).await {
Ok(dto) => dto,
Err(err) => {
warn!(
"Provider dns '{}' failed to read source.yml '{}': {err}",
provider.name,
source_path.display()
);
return;
}
};
let Some(provider_dtos) = sources_dto.provider.as_mut() else {
debug!(
"Provider dns '{}' source.yml '{}' has no provider section to persist resolved values",
provider.name,
source_path.display()
);
return;
};
let Some(provider_dto) = provider_dtos.iter_mut().find(|dto| dto.name.as_ref() == provider.name.as_ref()) else {
warn!(
"Provider dns '{}' not found in source.yml '{}', cannot persist resolved values",
provider.name,
source_path.display()
);
return;
};
let Some(dns_dto) = provider_dto.dns.as_mut() else {
debug!(
"Provider dns '{}' has no dns section in source.yml '{}', skipping resolved persist",
provider.name,
source_path.display()
);
return;
};
let resolved_hosts = {
let snapshot = provider.snapshot_resolved_ordered();
dns_dto.resolved = (!snapshot.is_empty()).then_some(snapshot);
dns_dto.resolved.as_ref().map_or(0, indexmap::IndexMap::len)
};
match write_sources_file_force(&source_path, &sources_dto).await {
Ok(()) => {
debug!(
"Provider dns '{}' persisted dns.resolved to '{}' (hosts={resolved_hosts})",
provider.name,
source_path.display()
);
}
Err(err) => {
warn!(
"Provider dns '{}' failed to persist dns.resolved to '{}': {err}",
provider.name,
source_path.display()
);
}
}
}
fn spawn_provider_dns_task(app_state: Arc<AppState>, provider_name: Arc<str>, cancel: CancellationToken) {
fn spawn_provider_dns_task(
app_state: Arc<AppState>,
provider_name: Arc<str>,
cancel: CancellationToken,
writer_tx: DnsResolvedWriteTx,
) {
tokio::spawn(async move {
let mut refresh_secs = 300_u64;
{
@@ -260,7 +143,7 @@ fn spawn_provider_dns_task(app_state: Arc<AppState>, provider_name: Arc<str>, ca
};
refresh_secs = provider.get_dns_config().map_or(300, |dns| dns.refresh_secs.max(10));
let stats = resolve_provider(&provider).await;
persist_provider_resolved_to_source_file(&app_state, &provider).await;
queue_provider_resolved_snapshot(&writer_tx, &provider).await;
let cache_hosts = provider.snapshot_resolved().len();
debug!(
"Provider dns tick '{}' finished: total_hosts={} resolved={} overridden={} empty={} failed={} cache_hosts={} elapsed_ms={}",
@@ -282,6 +165,7 @@ fn spawn_provider_dns_task(app_state: Arc<AppState>, provider_name: Arc<str>, ca
}
pub fn exec_provider_dns(app_state: &Arc<AppState>, cancel: &CancellationToken) {
let generation = next_dns_writer_generation();
let sources = app_state.app_config.sources.load();
let provider_names: Vec<_> = sources
.provider
@@ -292,13 +176,33 @@ pub fn exec_provider_dns(app_state: &Arc<AppState>, cancel: &CancellationToken)
drop(sources);
if provider_names.is_empty() {
let app_for_prune = Arc::clone(app_state);
tokio::spawn(async move {
prune_persisted_dns_resolved_to_runtime(&app_for_prune).await;
});
debug!("Provider dns manager: no enabled providers found");
return;
}
debug!("Provider dns manager: starting {} provider task(s)", provider_names.len());
let app_for_start = Arc::clone(app_state);
let cancel_for_start = cancel.clone();
tokio::spawn(async move {
// Seed runtime caches first, then start writer/tasks to avoid startup races.
load_persisted_dns_resolved(&app_for_start).await;
if cancel_for_start.is_cancelled() {
return;
}
for provider_name in provider_names {
spawn_provider_dns_task(Arc::clone(app_state), provider_name, cancel.clone());
}
let writer_tx = spawn_dns_resolved_writer(Arc::clone(&app_for_start), cancel_for_start.clone(), generation);
debug!("Provider dns manager: starting {} provider task(s)", provider_names.len());
for provider_name in provider_names {
spawn_provider_dns_task(
Arc::clone(&app_for_start),
provider_name,
cancel_for_start.clone(),
writer_tx.clone(),
);
}
});
}
backend/src/api/model/streams/provider_stream_factory.rs
+2 -1
View File
@@ -25,7 +25,6 @@ use reqwest::{
};
use shared::{
create_bitset,
model::{PlaylistItemType, DEFAULT_USER_AGENT},
utils::{filter_request_header, sanitize_sensitive_info},
};
use std::{
@@ -38,6 +37,8 @@ use std::{
time::{Duration, Instant},
};
use url::Url;
use shared::model::PlaylistItemType;
use shared::utils::DEFAULT_USER_AGENT;
const RETRY_SECONDS: u64 = 5;
const ERR_MAX_RETRY_COUNT: u32 = 5;