Feature/staged input and path improvements (#632)

* No template file not found error message
* Setup wizard improvmenets
* path improvements
* staged input fix
* path improvements
* Provider dns resolved ips now writing to a file instead of source.yml
* Playlist update hard kill after 1h
* matadata update tmdb fix
* Web UI fixes
* GeoIp update schedule
* batch:// prefix  for aliases
This commit is contained in:
euzu
2026-03-06 17:29:42 +01:00
committed by GitHub
parent c5632d0c98
commit 737d04d08b
82 changed files with 2506 additions and 1433 deletions
+95
View File
@@ -0,0 +1,95 @@
use crate::{
api::model::AppState,
model::InputSource,
repository::get_geoip_path,
utils::{request::download_text_content, GeoIp},
};
use log::{error, info};
use shared::{
model::{default_geoip_url, InputFetchMethod},
utils::Internable,
};
use std::{collections::HashMap, io::Cursor, sync::Arc};
#[derive(Debug)]
pub(crate) enum GeoIpUpdateError {
Disabled,
DownloadFailed(String),
ProcessFailed(String),
UnknownProcessing,
}
impl std::fmt::Display for GeoIpUpdateError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Disabled => write!(f, "GeoIp update is disabled"),
Self::DownloadFailed(err) => write!(f, "Failed to download geoip db: {err}"),
Self::ProcessFailed(err) => write!(f, "Failed to process geoip db: {err}"),
Self::UnknownProcessing => write!(f, "Unknown GeoIp processing error"),
}
}
}
impl std::error::Error for GeoIpUpdateError {}
pub(crate) async fn update_geoip_db(app_state: &Arc<AppState>) -> Result<(), GeoIpUpdateError> {
let config = app_state.app_config.config.load();
if let Some(geoip) = config.reverse_proxy.as_ref().and_then(|r| r.geoip.as_ref()) {
if geoip.enabled {
let geoip_db_path = &*get_geoip_path(&config.storage_dir);
let _file_lock = app_state.app_config.file_locks.write_lock(geoip_db_path).await;
let url = if geoip.url.trim().is_empty() { default_geoip_url() } else { geoip.url.clone() };
let input_source = InputSource {
name: "GeoIP".intern(),
url,
provider: None,
username: None,
password: None,
method: InputFetchMethod::GET,
headers: HashMap::default(),
};
return match download_text_content(
&app_state.app_config,
&app_state.http_client.load(),
&input_source,
None,
None,
false,
)
.await
{
Ok((content, _)) => {
let reader = Cursor::new(content);
let mut geoip = GeoIp::new();
let result = {
match geoip.import_ipv4_from_csv(reader, geoip_db_path) {
Ok(size) => (Some(size), None),
Err(err) => (None, Some(err)),
}
};
return match result {
(Some(_), None) => {
info!("GeoIp db updated");
app_state.geoip.store(Some(Arc::new(geoip)));
Ok(())
}
(None, Some(err)) => {
let error = GeoIpUpdateError::ProcessFailed(err.to_string());
error!("{error}");
Err(error)
}
_ => Err(GeoIpUpdateError::UnknownProcessing),
};
}
Err(err) => {
let error = GeoIpUpdateError::DownloadFailed(err.to_string());
error!("{error}");
Err(error)
}
};
}
}
Err(GeoIpUpdateError::Disabled)
}
+3 -3
View File
@@ -27,9 +27,9 @@ pub struct InputStatus {
pub clusters: HashMap<String, ClusterStatus>,
}
pub async fn resolve_input_storage_path(working_dir: &str, input_name: &str) -> PathBuf {
if let Ok(path) = get_input_storage_path(input_name, working_dir).await { path } else {
build_input_storage_path(input_name, working_dir)
pub async fn resolve_input_storage_path(storage_dir: &str, input_name: &str) -> PathBuf {
if let Ok(path) = get_input_storage_path(input_name, storage_dir).await { path } else {
build_input_storage_path(input_name, storage_dir)
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
pub(crate) mod geoip;
pub(crate) mod input_cache;
pub(crate) mod playlist_watch;
pub(crate) mod parser;
pub(crate) mod processor;
+1 -3
View File
@@ -123,14 +123,12 @@ impl TVGuide {
let src = "src".intern();
for child in children {
match child.name.as_ref() {
EPG_TAG_DISPLAY_NAME => {
if smart_match {
EPG_TAG_DISPLAY_NAME if smart_match => {
if let Some(name) = &child.value {
tag.normalized_epg_ids
.get_or_insert_with(Vec::new)
.push(normalize_channel_name(name, &id_cache.smart_match_config).intern());
}
}
}
EPG_TAG_ICON => {
if let Some(src) = child.get_attribute_value(&src) {
+1 -4
View File
@@ -174,9 +174,7 @@ pub async fn parse_xtream(input: &ConfigInput,
input.has_flag(ConfigInputFlags::XtreamLiveStreamWithoutExtension),
);
// Re-implement the loop to add ordinal
let mut ord_counter: u32 = 1;
for stream in xtream_streams {
for (ord_counter, stream) in (1_u32..).zip(xtream_streams) {
let group = group_map.get_mut(&stream.get_category_id()).unwrap_or(&mut unknown_grp);
let category_name = &group.category_name;
let stream_url = create_xtream_url(xtream_cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension);
@@ -200,7 +198,6 @@ pub async fn parse_xtream(input: &ConfigInput,
},
};
item.header.source_ordinal = ord_counter;
ord_counter += 1;
group.add(item);
}
+1 -1
View File
@@ -21,7 +21,7 @@ pub async fn process_group_watch(app_config: &Arc<AppConfig>, client: &reqwest::
let watch_filename = format!("{}/{}.bin", utils::sanitize_filename(target_name), utils::sanitize_filename(&pl.title));
let cfg = app_config.config.load();
match utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(&watch_filename))) {
match utils::get_file_path(&cfg.storage_dir, Some(std::path::PathBuf::from(&watch_filename))) {
Some(path) => {
let save_path = path.as_path();
let mut changed = false;
+66 -23
View File
@@ -50,13 +50,15 @@ use std::{
collections::{HashMap, HashSet},
path::PathBuf,
sync::{Arc, Weak},
time::Instant,
time::{Duration, Instant},
};
use tokio::{
sync::{Mutex, OwnedRwLockWriteGuard, RwLock},
task::JoinSet,
};
const PLAYLIST_UPDATE_MAX_DURATION_SECS: u64 = 3600;
fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool {
let provider = ValueProvider { pli, match_as_ascii };
filter.filter(&provider)
@@ -324,10 +326,11 @@ async fn playlist_download_from_input(
input: &ConfigInput,
) -> PlaylistDownloadResult {
let config = &*app_config.config.load();
let working_dir = &config.working_dir;
let storage_dir = &config.storage_dir;
let download_input_type = input.get_download_input_type();
// Check Status
let storage_path = input_cache::resolve_input_storage_path(working_dir, &input.name).await;
let storage_path = input_cache::resolve_input_storage_path(storage_dir, &input.name).await;
let mut status = input_cache::load_input_status(&storage_path);
let cache_duration = input.cache_duration_seconds;
@@ -336,7 +339,7 @@ async fn playlist_download_from_input(
let _ = std::fs::create_dir_all(&storage_path);
}
let (clusters_to_download, fully_cached) = match input.input_type {
let (clusters_to_download, fully_cached) = match download_input_type {
InputType::Xtream => {
let mut to_download = vec![];
for c in XTREAM_CLUSTER {
@@ -364,7 +367,7 @@ async fn playlist_download_from_input(
return PlaylistDownloadResult::new(vec![], vec![], true, false);
}
let (playlist, errors, persisted) = match input.input_type {
let (playlist, errors, persisted) = match download_input_type {
InputType::M3u => {
let (p, e) = m3u::download_m3u_playlist(app_config, client, config, input).await;
(p, e, false)
@@ -381,7 +384,7 @@ async fn playlist_download_from_input(
// Update Status
if errors.is_empty() {
if let InputType::Xtream = input.input_type {
if let InputType::Xtream = download_input_type {
if let Some(clusters) = clusters_to_download {
for c in clusters {
input_cache::update_cluster_status(&mut status, &c.to_string(), ClusterState::Ok);
@@ -401,7 +404,7 @@ async fn playlist_download_from_input(
// We could mark specific clusters as failed if we knew which one failed.
// For simplicity, if error, we don't update the timestamp (so it stays expired/invalid).
// Or we mark as Failed.
if let InputType::Xtream = input.input_type {
if let InputType::Xtream = download_input_type {
if let Some(clusters) = clusters_to_download {
for c in clusters {
// Optimistic: Only mark failed if we are sure?
@@ -437,6 +440,7 @@ async fn process_source(
continue;
};
if is_input_enabled(input, &ctx.user_targets) {
let effective_input_type = input.get_download_input_type();
source_downloaded = true;
log_memory_snapshot(format!("source[{source_idx}] input '{}' before_download", input.name).as_str());
@@ -459,7 +463,7 @@ async fn process_source(
};
log_memory_snapshot(format!("source[{source_idx}] input '{}' after_download", input.name).as_str());
let tvguide = if input.input_type == InputType::Library {
let tvguide = if effective_input_type == InputType::Library {
None
} else {
download_input_epg(ctx, input, &mut error_list).await
@@ -483,7 +487,7 @@ async fn process_source(
let elapsed = start_time.elapsed().as_secs();
input_stats.insert(
input_name.clone(),
create_input_stat(group_count, channel_count, errors.len(), input.input_type, input_name, elapsed),
create_input_stat(group_count, channel_count, errors.len(), effective_input_type, input_name, elapsed),
);
}
}
@@ -547,8 +551,8 @@ async fn download_input_epg(
) -> Option<TVGuide> {
// Download epg for input
let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
let working_dir = &ctx.config.config.load().working_dir;
epg::get_xmltv(ctx, input, None, working_dir).await
let storage_dir = &ctx.config.config.load().storage_dir;
epg::get_xmltv(ctx, input, None, storage_dir).await
} else {
(None, vec![])
};
@@ -1018,10 +1022,10 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
}
let input_name = fpl.input.name.clone();
let input_type = fpl.input.input_type;
let xtream_probe_handled = input_type.is_xtream() && target.get_xtream_output().is_some();
let effective_input_type = fpl.input.get_download_input_type();
let xtream_probe_handled = effective_input_type.is_xtream() && target.get_xtream_output().is_some();
let live_probe_settings = if probe_live_enabled {
get_live_probe_interval_settings(target, input_type, Some(opts)).map(|(delay, interval_secs)| {
get_live_probe_interval_settings(target, effective_input_type, Some(opts)).map(|(delay, interval_secs)| {
let interval_signed = i64::try_from(interval_secs).unwrap_or(i64::MAX);
let cutoff_ts = chrono::Utc::now().timestamp().saturating_sub(interval_signed);
(delay, interval_secs, cutoff_ts)
@@ -1050,6 +1054,16 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
if needs_live_probe(&item, cutoff_ts) {
if let Some(provider_id) = provider_id_from_item(&item) {
if queued_live_keys.insert(provider_id.clone()) {
if log_enabled!(Level::Debug) {
let last_probed = match item.header.additional_properties.as_ref() {
Some(StreamProperties::Live(props)) => props.last_probed_timestamp,
_ => None,
};
debug!(
"[Task] Creating ProbeLive task for input {}: id={}, last_probed_ts={:?}, cutoff_ts={}, interval={}s, title=\"{}\"",
input_name, provider_id, last_probed, cutoff_ts, interval_secs, item.header.title
);
}
let task = UpdateTask::ProbeLive {
id: provider_id,
reason: ResolveReasonSet::from_variants(&[ResolveReason::Probe]),
@@ -1092,7 +1106,7 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
}
// For M3U, ID is a provider id; for Library, ID is UUID.
let unique_id = if input_type == InputType::Library {
let unique_id = if effective_input_type == InputType::Library {
item.header.uuid.to_valid_uuid()
} else {
item.header.id.to_string()
@@ -1104,6 +1118,10 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
continue;
}
debug!(
"[Task] Creating ProbeStream task for input {}: scope={}, unique_id={}, item_type={:?}, title=\"{}\"",
input_name, probe_scope, unique_id, item.header.item_type, item.header.title
);
let task = UpdateTask::ProbeStream {
probe_scope,
unique_id,
@@ -1204,7 +1222,7 @@ async fn process_watch(
}
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
pub async fn exec_processing(
client: &reqwest::Client,
app_config: Arc<AppConfig>,
@@ -1219,6 +1237,7 @@ pub async fn exec_processing(
pre_processed_inputs: Option<HashSet<Arc<str>>>,
acquired_permit: Option<crate::api::model::UpdateGuardPermit>,
) {
let max_update_duration = Duration::from_secs(PLAYLIST_UPDATE_MAX_DURATION_SECS);
let playlist_guard = if let Some(permit) = acquired_permit {
Some(permit)
} else if let Some(guard) = &update_guard {
@@ -1237,7 +1256,15 @@ pub async fn exec_processing(
if playlist_guard.is_some() {
if let Some(state) = app_state.as_ref() {
sync_panel_api_exp_dates(state).await;
if tokio::time::timeout(max_update_duration, sync_panel_api_exp_dates(state)).await.is_err() {
error!(
"Playlist update bootstrap timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
);
if let Some(events) = event_manager.as_deref() {
events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
}
return;
}
}
}
@@ -1266,13 +1293,29 @@ pub async fn exec_processing(
};
let start_time = Instant::now();
let process_result = std::panic::AssertUnwindSafe(process_sources(&ctx)).catch_unwind().await;
let Ok((stats, errors)) = process_result else {
error!("Playlist processing panicked");
if let Some(events) = event_manager.as_deref() {
events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
let process_result = tokio::time::timeout(
max_update_duration,
std::panic::AssertUnwindSafe(process_sources(&ctx)).catch_unwind(),
)
.await;
let (stats, errors) = match process_result {
Ok(Ok((stats, errors))) => (stats, errors),
Ok(Err(_)) => {
error!("Playlist processing panicked");
if let Some(events) = event_manager.as_deref() {
events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
}
return;
}
Err(_) => {
error!(
"Playlist processing timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
);
if let Some(events) = event_manager.as_deref() {
events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
}
return;
}
return;
};
log_memory_snapshot("exec_processing after_process_sources");
@@ -41,7 +41,7 @@ pub async fn update_generic_stream_metadata(
active_provider: &Arc<ActiveProviderManager>,
active_handle: Option<&crate::api::model::ProviderHandle>,
) -> Result<GenericProbeOutcome, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
let storage_dir = &app_config.config.load().storage_dir;
// Check if probing is enabled globally
let ffprobe_enabled = app_config.is_ffprobe_enabled().await;
@@ -50,7 +50,7 @@ pub async fn update_generic_stream_metadata(
}
// Determine storage file path based on input type
let storage_path = get_input_storage_path(&input.name, working_dir).await
let storage_path = get_input_storage_path(&input.name, storage_dir).await
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
let (db_path, storage_kind) = match input.input_type {
+2 -2
View File
@@ -21,8 +21,8 @@ pub async fn update_live_stream_metadata(
_active_handle: Option<&ProviderHandle>,
_active_provider: &Arc<ActiveProviderManager>,
) -> Result<Option<LiveStreamProperties>, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
let storage_path = get_input_storage_path(&input.name, working_dir).await
let storage_dir = &app_config.config.load().storage_dir;
let storage_path = get_input_storage_path(&input.name, storage_dir).await
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
// Try to load existing info first to preserve data
@@ -207,6 +207,19 @@ fn queue_background_series_info(
let reasons = check_resolve_reasons(resolve_options, resolve_tmdb_enabled, pli);
if !reasons.is_empty() {
if log_enabled!(Level::Debug) {
let has_details = pli.has_details();
let (has_tmdb, has_date) = match pli.header.additional_properties.as_ref() {
Some(StreamProperties::Series(props)) => {
(props.tmdb.is_some(), props.release_date.is_some())
}
_ => (false, false),
};
debug!(
"[Task] Creating ResolveSeries task for input {}: id={}, reasons={}, has_details={}, has_tmdb={}, has_date={}, title=\"{}\"",
input_name_arc, provider_id, reasons, has_details, has_tmdb, has_date, pli.header.title
);
}
let task = UpdateTask::ResolveSeries {
id: provider_id,
reason: reasons,
@@ -233,8 +246,8 @@ async fn process_immediate_series_info(
resolve_tmdb_enabled: bool,
) -> Vec<PlaylistGroup> {
let input = fpl.input;
let working_dir = &ctx.config.config.load().working_dir;
let storage_path = match get_input_storage_path(&input.name, working_dir).await {
let storage_dir = &ctx.config.config.load().storage_dir;
let storage_path = match get_input_storage_path(&input.name, storage_dir).await {
Ok(path) => path,
Err(err) => {
error!("Can't resolve series, input storage directory for input '{}' failed: {err}", input.name);
@@ -624,8 +637,8 @@ pub async fn update_series_metadata(
db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
tmdb_and_date_present_out: Option<&AtomicBool>,
) -> Result<Option<SeriesStreamProperties>, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
let storage_path = get_input_storage_path(&input.name, working_dir)
let storage_dir = &app_config.config.load().storage_dir;
let storage_path = get_input_storage_path(&input.name, storage_dir)
.await
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
+25 -13
View File
@@ -5,6 +5,7 @@ use crate::model::FetchedPlaylist;
use crate::model::InputSource;
use crate::model::{AppConfig, ConfigTarget};
use crate::model::{ConfigInput, ConfigInputFlags};
use crate::processing::input_cache::resolve_input_storage_path;
use crate::processing::processor::playlist::PlaylistProcessingContext;
use crate::processing::processor::{
create_resolve_options_function_for_xtream_target, process_foreground_retry_once, select_cancel_token,
@@ -12,7 +13,6 @@ use crate::processing::processor::{
FOREGROUND_RETRY_BATCH_MAX_SIZE as RETRY_BATCH_MAX_SIZE,
};
use crate::ptt::ptt_parse_title;
use crate::repository::get_input_storage_path;
use crate::repository::persist_input_vod_info;
use crate::repository::persist_input_vod_info_batch;
use crate::repository::{xtream_get_file_path, BPlusTreeQuery};
@@ -148,15 +148,9 @@ async fn process_immediate_vod_info(
resolve_tmdb_enabled: bool,
) {
let input = fpl.input;
let working_dir = &ctx.config.config.load().working_dir;
let storage_dir = &ctx.config.config.load().storage_dir;
let storage_path = match get_input_storage_path(&input.name, working_dir).await {
Ok(path) => path,
Err(err) => {
error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name);
return;
}
};
let storage_path = resolve_input_storage_path(storage_dir, &input.name).await;
// Keep an optional read query open and reopen lazily only when needed.
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Video);
@@ -447,6 +441,26 @@ fn queue_background_vod_info(
let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli);
if !reasons.is_empty() {
if log_enabled!(Level::Debug) {
let has_details = pli.has_details();
let (has_tmdb, has_date, has_video, has_audio) =
match pli.header.additional_properties.as_ref() {
Some(StreamProperties::Video(props)) => {
let details = props.details.as_ref();
(
props.tmdb.is_some(),
details.and_then(|d| d.release_date.as_ref()).is_some(),
MediaQuality::is_valid_media_info(details.and_then(|d| d.video.as_deref())),
MediaQuality::is_valid_media_info(details.and_then(|d| d.audio.as_deref())),
)
}
_ => (false, false, false, false),
};
debug!(
"[Task] Creating ResolveVod task for input {}: id={}, reasons={}, has_details={}, has_tmdb={}, has_date={}, has_video_info={}, has_audio_info={}, title=\"{}\"",
input.name, provider_id, reasons, has_details, has_tmdb, has_date, has_video, has_audio, pli.header.title
);
}
let task =
UpdateTask::ResolveVod { id: provider_id, reason: reasons, delay: resolve_options.resolve_delay };
mgr.queue_task_background(input.name.clone(), task);
@@ -509,10 +523,8 @@ pub async fn update_vod_metadata(
db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
tmdb_resolved_out: Option<&AtomicBool>,
) -> Result<Option<VideoStreamProperties>, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
let storage_path = get_input_storage_path(&input.name, working_dir)
.await
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
let storage_dir = &app_config.config.load().storage_dir;
let storage_path = resolve_input_storage_path(storage_dir, &input.name).await;
// Check if we should skip based on input options
if input.has_flag(ConfigInputFlags::XtreamSkipVod) {