euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-20 06:42:20 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

MetadatResolve hardening rerties after errors (#611)

Browse Source 
* MetadatResolve hardening rerties after errors
This commit is contained in:
euzu
2026-02-25 19:30:19 +01:00
committed by GitHub
parent b3b0ccd92b
commit 3be0fa347f
36 changed files with 3164 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/processing/processor/mod.rs
+171
View File
@@ -43,6 +43,10 @@ impl Default for ResolveOptions {
}
}
pub(crate) const FOREGROUND_BATCH_SIZE: usize = 200;
pub(crate) const FOREGROUND_RETRY_BATCH_MAX_SIZE: usize = FOREGROUND_BATCH_SIZE * 4;
pub(crate) const FOREGROUND_MIN_RETRY_DELAY_SECS: u64 = 1;
//
// fn get_resolve_<cluster>_options(target: &ConfigTarget, fpl: &FetchedPlaylist) -> bool
@@ -110,4 +114,171 @@ macro_rules! create_resolve_options_function_for_xtream_target {
};
}
use create_resolve_options_function_for_xtream_target;
/// Foreground retry helper that retries each queued item at most once.
///
/// `retry_delay_secs` is applied sequentially per item, so the total wall-clock
/// delay is roughly `retry_delay_secs * retry_item_count` plus network/DB time.
macro_rules! process_foreground_retry_once {
(
ctx: $ctx:expr,
fpl: $fpl:expr,
filter: $filter:expr,
retry_once_ids: $retry_once_ids:ident,
retry_delay_secs: $retry_delay_secs:expr,
xtream_path: $xtream_path:ident,
db_query_holder: $db_query_holder:ident,
db_lock_holder: $db_lock_holder:ident,
batch: $batch:ident,
batch_size: $batch_size:expr,
retry_batch_max_len: $retry_batch_max_len:expr,
processed_count: $processed_count:ident,
query_error_context: $query_error_context:expr,
reasons: |$pli_reasons:ident| $reasons_expr:expr,
update: |$active_provider:ident, $pli_update:ident, $provider_id:ident, $reasons:ident, $db_query_ref:ident| $update_expr:expr,
apply_properties: |$pli_apply:ident, $updated_props:ident| $apply_expr:expr,
persist: |$updates:ident| $persist_expr:expr,
on_persist_error: |$persist_err:ident| $on_persist_error_expr:expr,
on_retry_error: |$pli_error:ident, $retry_err:ident| $on_retry_error_expr:expr,
on_after_attempt: |$pli_after:ident, $retry_succeeded:ident| $on_after_attempt_expr:expr $(,)?
) => {
for __pli in $fpl.items_mut() {
if !($filter)(__pli) {
continue;
}
let __provider_id = if let Ok(__uid) = __pli.header.id.parse::<u32>() {
crate::api::model::ProviderIdType::Id(__uid)
} else {
crate::api::model::ProviderIdType::from(&*__pli.header.id)
};
if !$retry_once_ids.remove(&__provider_id) {
continue;
}
let mut __retry_succeeded = false;
let __reasons = {
let $pli_reasons = &mut *__pli;
$reasons_expr
};
if !__reasons.is_empty() {
if let Some(__active_provider) = $ctx.provider_manager.as_ref() {
// Do not hold a read lock over the retry delay window.
if $db_query_holder.is_some() {
$db_query_holder = None;
$db_lock_holder = None;
}
tokio::time::sleep(std::time::Duration::from_secs($retry_delay_secs)).await;
if $db_query_holder.is_none() && $xtream_path.exists() {
let __file_lock = $ctx.config.file_locks.read_lock(&$xtream_path).await;
let __xtream_path = $xtream_path.clone();
let __query = match tokio::task::spawn_blocking(move || {
crate::repository::BPlusTreeQuery::<u32, shared::model::XtreamPlaylistItem>::try_new(
&__xtream_path,
)
})
.await
{
Ok(Ok(__query)) => Some((__query, __file_lock)),
Ok(Err(__err)) => {
log::error!("Failed to open BPlusTreeQuery for {}: {__err}", $query_error_context);
None
}
Err(__err) => {
log::error!("Failed to open BPlusTreeQuery for {}: {__err}", $query_error_context);
None
}
};
if let Some((__query, __guard)) = __query {
$db_query_holder = Some(std::sync::Arc::new(parking_lot::Mutex::new(__query)));
$db_lock_holder = Some(__guard);
}
}
let __db_query_ref = $db_query_holder.as_ref().map(std::sync::Arc::clone);
let __update_future = {
let $active_provider = __active_provider;
let $pli_update = &mut *__pli;
let $provider_id = __provider_id.clone();
let $reasons = &__reasons;
let $db_query_ref = __db_query_ref;
$update_expr
};
match __update_future.await {
Ok(Some(__updated_props)) => {
{
let $pli_apply = &mut *__pli;
let $updated_props = &__updated_props;
$apply_expr
}
$batch.push((__provider_id.clone(), __updated_props));
if $batch.len() >= $batch_size {
$db_query_holder = None;
$db_lock_holder = None;
let __updates: Vec<(u32, _)> = $batch
.iter()
.filter_map(|(__id, __props)| {
// Foreground retry batches can include text provider IDs (e.g. M3U).
// Persist batch functions for these paths are keyed by numeric Xtream IDs,
// so text IDs are intentionally skipped here.
if let crate::api::model::ProviderIdType::Id(__vid) = __id {
Some((*__vid, __props.clone()))
} else {
None
}
})
.collect();
if __updates.is_empty() {
$batch.clear();
} else {
let __persist_future = {
let $updates = __updates;
$persist_expr
};
match __persist_future.await {
Ok(()) => $batch.clear(),
Err($persist_err) => {
$on_persist_error_expr;
if $batch.len() > $retry_batch_max_len {
let __drop_count = $batch.len().saturating_sub($retry_batch_max_len);
if __drop_count > 0 {
$batch.drain(0..__drop_count);
}
}
}
}
}
}
$processed_count += 1;
__retry_succeeded = true;
}
Ok(None) => {}
Err($retry_err) => {
let $pli_error = &*__pli;
$on_retry_error_expr;
}
}
}
}
{
let $pli_after = &mut *__pli;
let $retry_succeeded = __retry_succeeded;
$on_after_attempt_expr;
}
}
};
}
pub(crate) use process_foreground_retry_once;
use shared::utils::default_resolve_delay_secs;
backend/src/processing/processor/stream_probe.rs
+114 -87
View File
@@ -15,6 +15,13 @@ enum ProbeStorageKind {
Xtream,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GenericProbeOutcome {
Updated,
Noop,
ProbeFailed,
}
fn requires_provider_connection_for_generic_probe(input_type: InputType) -> bool {
!matches!(input_type, InputType::Library)
}
@@ -31,13 +38,13 @@ pub async fn update_generic_stream_metadata(
item_type: PlaylistItemType,
active_provider: &Arc<ActiveProviderManager>,
active_handle: Option<&crate::api::model::ProviderHandle>,
) -> Result<(), TuliproxError> {
) -> Result<GenericProbeOutcome, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
// Check if probing is enabled globally
let ffprobe_enabled = app_config.is_ffprobe_enabled().await;
if !ffprobe_enabled {
return Ok(());
return Ok(GenericProbeOutcome::Noop);
}
// Determine storage file path based on input type
@@ -62,7 +69,7 @@ pub async fn update_generic_stream_metadata(
XtreamCluster::Series
} else {
// Generic probing currently supports live/video/series payload shapes.
return Ok(());
return Ok(GenericProbeOutcome::Noop);
};
(
xtream_get_file_path(&storage_path, cluster),
@@ -90,10 +97,20 @@ pub async fn update_generic_stream_metadata(
let probe_url = stream_url.to_string();
let config = app_config.config.load();
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
let metadata_update = config.metadata_update.clone().unwrap_or_default();
let ffprobe_timeout = metadata_update.ffprobe_timeout.unwrap_or(60);
let user_agent = config.default_user_agent.clone();
let analyze_duration = 10_000_000;
let probe_size = 10_000_000;
let (analyze_duration, probe_size) = if item_type.is_live() {
(
metadata_update.ffprobe_live_analyze_duration_micros,
metadata_update.ffprobe_live_probe_size_bytes,
)
} else {
(
metadata_update.ffprobe_analyze_duration_micros,
metadata_update.ffprobe_probe_size_bytes,
)
};
debug_if_enabled!("Probing Generic Stream '{unique_id}'");
@@ -111,96 +128,106 @@ pub async fn update_generic_stream_metadata(
let Some((_quality, raw_video, raw_audio)) = probe_data else {
warn!("Probe failed or timed out for generic stream: {unique_id}");
return Ok(());
return Ok(GenericProbeOutcome::ProbeFailed);
};
// Acquire lock and open a tree for update
let _file_lock = app_config.file_locks.write_lock(&db_path).await;
// Hold the async file lock while the blocking DB update runs in a blocking thread.
let file_lock = app_config.file_locks.write_lock(&db_path).await;
let db_path_for_update = db_path.clone();
let unique_id_for_update = unique_id.to_string();
let updated = tokio::task::spawn_blocking(move || -> Result<bool, String> {
let mut updated = false;
match storage_kind {
ProbeStorageKind::M3u => {
let key: Arc<str> = Arc::from(unique_id_for_update.as_str());
let mut tree_update = BPlusTreeUpdate::<Arc<str>, M3uPlaylistItem>::try_new(&db_path_for_update)
.map_err(|e| format!("Failed to open M3U tree update: {e}"))?;
// Update the record in BPlusTree
match storage_kind {
ProbeStorageKind::M3u => {
let key: Arc<str> = unique_id.into();
let mut tree_update = BPlusTreeUpdate::<Arc<str>, M3uPlaylistItem>::try_new(&db_path)
.map_err(|e| shared::error::info_err!("Failed to open M3U tree update: {e}"))?;
if let Some(mut item) = tree_update.query(&key).map_err(|e| format!("Tree query error: {e}"))? {
update_properties(
&mut item.additional_properties,
item_type,
&item.name,
item.virtual_id,
raw_video,
raw_audio,
);
tree_update
.update(&key, item)
.map_err(|e| format!("Tree update error: {e}"))?;
info!("Successfully updated M3U metadata for: {unique_id_for_update}");
updated = true;
} else {
warn!("Item not found in M3U DB: {unique_id_for_update}");
}
}
ProbeStorageKind::Library => {
let mut tree_update = BPlusTreeUpdate::<UUIDType, XtreamPlaylistItem>::try_new(&db_path_for_update)
.map_err(|e| format!("Failed to open Library tree update: {e}"))?;
let uuid = UUIDType::from_valid_uuid(&unique_id_for_update);
if let Some(mut item) = tree_update
.query(&key)
.map_err(|e| shared::error::info_err!("Tree query error: {e}"))?
{
update_properties(
&mut item.additional_properties,
item_type,
&item.name,
item.virtual_id,
raw_video,
raw_audio,
);
tree_update
.update(&key, item)
.map_err(|e| shared::error::info_err!("Tree update error: {e}"))?;
info!("Successfully updated M3U metadata for: {unique_id}");
} else {
warn!("Item not found in M3U DB: {unique_id}");
if let Some(mut item) = tree_update.query(&uuid).map_err(|e| format!("Tree query error: {e}"))? {
update_properties(
&mut item.additional_properties,
item_type,
&item.name,
item.virtual_id,
raw_video,
raw_audio,
);
tree_update
.update(&uuid, item)
.map_err(|e| format!("Tree update error: {e}"))?;
info!("Successfully updated Library metadata for: {unique_id_for_update}");
updated = true;
} else {
warn!("Item not found in Library DB: {unique_id_for_update}");
}
}
ProbeStorageKind::Xtream => {
let Ok(provider_id) = unique_id_for_update.parse::<u32>() else {
warn!("Skipping xtream generic probe update with non-numeric id: {unique_id_for_update}");
return Ok(false);
};
let mut tree_update = BPlusTreeUpdate::<u32, XtreamPlaylistItem>::try_new(&db_path_for_update)
.map_err(|e| format!("Failed to open Xtream tree update: {e}"))?;
if let Some(mut item) = tree_update
.query(&provider_id)
.map_err(|e| format!("Tree query error: {e}"))?
{
update_properties(
&mut item.additional_properties,
item_type,
&item.name,
item.virtual_id,
raw_video,
raw_audio,
);
tree_update
.update(&provider_id, item)
.map_err(|e| format!("Tree update error: {e}"))?;
info!("Successfully updated Xtream metadata for: {unique_id_for_update}");
updated = true;
} else {
warn!("Item not found in Xtream DB: {unique_id_for_update}");
}
}
}
ProbeStorageKind::Library => {
let mut tree_update = BPlusTreeUpdate::<UUIDType, XtreamPlaylistItem>::try_new(&db_path)
.map_err(|e| shared::error::info_err!("Failed to open Library tree update: {e}"))?;
let uuid = UUIDType::from_valid_uuid(unique_id);
if let Some(mut item) = tree_update
.query(&uuid)
.map_err(|e| shared::error::info_err!("Tree query error: {e}"))?
{
update_properties(
&mut item.additional_properties,
item_type,
&item.name,
item.virtual_id,
raw_video,
raw_audio,
);
tree_update
.update(&uuid, item)
.map_err(|e| shared::error::info_err!("Tree update error: {e}"))?;
info!("Successfully updated Library metadata for: {unique_id}");
} else {
warn!("Item not found in Library DB: {unique_id}");
}
}
ProbeStorageKind::Xtream => {
let Ok(provider_id) = unique_id.parse::<u32>() else {
warn!("Skipping xtream generic probe update with non-numeric id: {unique_id}");
return Ok(());
};
Ok(updated)
})
.await
.map_err(|e| shared::error::info_err!("Failed to join generic probe DB update task: {e}"))?
.map_err(|e| shared::error::info_err!("{e}"))?;
let mut tree_update = BPlusTreeUpdate::<u32, XtreamPlaylistItem>::try_new(&db_path)
.map_err(|e| shared::error::info_err!("Failed to open Xtream tree update: {e}"))?;
if let Some(mut item) = tree_update
.query(&provider_id)
.map_err(|e| shared::error::info_err!("Tree query error: {e}"))?
{
update_properties(
&mut item.additional_properties,
item_type,
&item.name,
item.virtual_id,
raw_video,
raw_audio,
);
tree_update
.update(&provider_id, item)
.map_err(|e| shared::error::info_err!("Tree update error: {e}"))?;
info!("Successfully updated Xtream metadata for: {unique_id}");
} else {
warn!("Item not found in Xtream DB: {unique_id}");
}
}
drop(file_lock);
if updated {
Ok(GenericProbeOutcome::Updated)
} else {
Ok(GenericProbeOutcome::Noop)
}
Ok(())
}
fn update_properties(
backend/src/processing/processor/xtream.rs
+7 -4
View File
@@ -7,7 +7,7 @@ use crate::repository::{get_input_storage_path, persist_input_live_info, BPlusTr
use crate::utils::{debug_if_enabled};
use log::{debug, warn};
use crate::processing::parser::xtream::create_xtream_url;
use crate::api::model::ProviderIdType;
use crate::api::model::{ActiveProviderManager, ProviderHandle, ProviderIdType};
/// Updates metadata for a single Live stream (primarily probing)
#[allow(clippy::too_many_lines)]
@@ -17,6 +17,8 @@ pub async fn update_live_stream_metadata(
id: ProviderIdType,
save: bool,
db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
_active_handle: Option<&ProviderHandle>,
_active_provider: &Arc<ActiveProviderManager>,
) -> Result<Option<LiveStreamProperties>, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
let storage_path = get_input_storage_path(&input.name, working_dir).await
@@ -114,10 +116,11 @@ pub async fn update_live_stream_metadata(
use_prefix, no_ext
);
let config = app_config.config.load();
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
let metadata_update = config.metadata_update.clone().unwrap_or_default();
let ffprobe_timeout = metadata_update.ffprobe_timeout.unwrap_or(60);
let user_agent = config.default_user_agent.clone();
let analyze_duration = 5_000_000;
let probe_size = 5_000_000;
let analyze_duration = metadata_update.ffprobe_live_analyze_duration_micros;
let probe_size = metadata_update.ffprobe_live_probe_size_bytes;
let display_id = stream_id_opt.map_or_else(|| "StringID".to_string(), |v| v.to_string());
debug!("Probing Live Stream ID {} for input {}", display_id, input.name);
backend/src/processing/processor/xtream_series.rs
+132 -40
View File
@@ -2,13 +2,15 @@ use crate::api::model::UpdateTask;
use crate::api::model::{ActiveProviderManager, ProviderHandle, ProviderIdType, ResolveReason, ResolveReasonSet};
use crate::library::MetadataResolver;
use crate::model::FetchedPlaylist;
use crate::model::{AppConfig, ConfigTarget};
use crate::model::{AppConfig, ConfigTarget, MetadataUpdateConfig};
use crate::model::{ConfigInput, ConfigInputFlags, InputSource};
use crate::processing::parser::xtream::create_xtream_series_episode_url;
use crate::processing::parser::xtream::parse_xtream_series_info;
use crate::processing::processor::playlist::{PlaylistProcessingContext, ProcessingPipe};
use crate::processing::processor::{
create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags,
create_resolve_options_function_for_xtream_target, process_foreground_retry_once, ResolveOptions,
ResolveOptionsFlags, FOREGROUND_BATCH_SIZE as BATCH_SIZE, FOREGROUND_MIN_RETRY_DELAY_SECS,
FOREGROUND_RETRY_BATCH_MAX_SIZE as RETRY_BATCH_MAX_SIZE,
};
use crate::ptt::ptt_parse_title;
use crate::repository::persists_input_series_info;
@@ -26,13 +28,31 @@ use shared::model::{
XtreamSeriesInfo,
};
use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::{Duration, Instant};
create_resolve_options_function_for_xtream_target!(series);
const BATCH_SIZE: usize = 100;
#[derive(Debug, Clone, Copy)]
pub struct SeriesProbeSettings {
pub timeout_secs: u64,
pub analyze_duration_micros: u64,
pub probe_size_bytes: u64,
}
impl SeriesProbeSettings {
pub fn from_metadata_update(metadata_update: Option<&MetadataUpdateConfig>) -> Self {
let defaults = MetadataUpdateConfig::default();
Self {
timeout_secs: metadata_update.and_then(|cfg| cfg.ffprobe_timeout).unwrap_or(60),
analyze_duration_micros: metadata_update
.map_or(defaults.ffprobe_analyze_duration_micros, |cfg| cfg.ffprobe_analyze_duration_micros),
probe_size_bytes: metadata_update
.map_or(defaults.ffprobe_probe_size_bytes, |cfg| cfg.ffprobe_probe_size_bytes),
}
}
}
#[allow(clippy::too_many_arguments)]
pub async fn playlist_resolve_series(
@@ -227,8 +247,13 @@ async fn process_immediate_series_info(
let mut groups_to_add = Vec::new();
let mut batch: Vec<(ProviderIdType, SeriesStreamProperties)> = Vec::with_capacity(BATCH_SIZE);
let mut retry_once_ids: HashSet<ProviderIdType> = HashSet::new();
let mut processed_count = 0;
let mut last_log_time = Instant::now();
let series_probe_settings = {
let config = ctx.config.config.load();
SeriesProbeSettings::from_metadata_update(config.metadata_update.as_ref())
};
for pli in fpl.items_mut() {
if !filter(pli) {
@@ -240,6 +265,7 @@ async fn process_immediate_series_info(
} else {
ProviderIdType::from(&*pli.header.id)
};
let mut defer_expand = false;
if !skip_resolve {
let reasons = check_resolve_reasons(resolve_options, resolve_tmdb_enabled, pli);
@@ -250,13 +276,11 @@ async fn process_immediate_series_info(
let file_lock = ctx.config.file_locks.read_lock(&xtream_path).await;
let xtream_path = xtream_path.clone();
let query = match tokio::task::spawn_blocking(move || {
let guard = file_lock;
let query = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path)?;
Ok::<_, std::io::Error>((query, guard))
BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path)
})
.await
{
Ok(Ok((query, guard))) => Some((query, guard)),
Ok(Ok(query)) => Some((query, file_lock)),
Ok(Err(err)) => {
error!("Failed to open BPlusTreeQuery for Series: {err}");
None
@@ -283,6 +307,7 @@ async fn process_immediate_series_info(
provider_id.clone(),
&reasons,
db_query_ref,
series_probe_settings,
)
.await
{
@@ -338,14 +363,18 @@ async fn process_immediate_series_info(
Ok(None) => {}
Err(e) => {
error!("Failed to update Series metadata for {}: {e}", pli.header.title);
retry_once_ids.insert(provider_id.clone());
defer_expand = true;
}
}
}
}
}
if let Some(group_obj) = expand_series_item(pli, input) {
groups_to_add.push(group_obj);
if !defer_expand {
if let Some(group_obj) = expand_series_item(pli, input) {
groups_to_add.push(group_obj);
}
}
if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 {
@@ -354,6 +383,64 @@ async fn process_immediate_series_info(
}
}
if !retry_once_ids.is_empty() {
let retry_delay_secs = u64::from(resolve_options.resolve_delay).max(FOREGROUND_MIN_RETRY_DELAY_SECS);
process_foreground_retry_once!(
ctx: ctx,
fpl: fpl,
filter: filter,
retry_once_ids: retry_once_ids,
retry_delay_secs: retry_delay_secs,
xtream_path: xtream_path,
db_query_holder: db_query_holder,
db_lock_holder: _db_lock_holder,
batch: batch,
batch_size: BATCH_SIZE,
retry_batch_max_len: RETRY_BATCH_MAX_SIZE,
processed_count: processed_count,
query_error_context: "Series retry",
reasons: |retry_pli| {
if skip_resolve {
ResolveReasonSet::new()
} else {
check_resolve_reasons(resolve_options, resolve_tmdb_enabled, retry_pli)
}
},
update: |active_provider, retry_pli, provider_id, reasons, db_query_ref| update_series_info_immediate(
ctx,
active_provider,
input,
retry_pli,
provider_id,
reasons,
db_query_ref,
series_probe_settings,
),
apply_properties: |retry_pli, updated_props| {
retry_pli.header.additional_properties =
Some(StreamProperties::Series(Box::new(updated_props.clone())));
},
persist: |updates| persist_input_series_info_batch(
&ctx.config,
&storage_path,
XtreamCluster::Series,
&input.name,
updates,
),
on_persist_error: |err| {
error!("persist_input_series_info_batch failed for Series retry on input '{}'. Error: {}", input.name, err);
},
on_retry_error: |retry_pli, err| {
error!("Foreground retry failed for Series {}: {err}", retry_pli.header.title);
},
on_after_attempt: |retry_pli, _retry_succeeded| {
if let Some(group_obj) = expand_series_item(retry_pli, input) {
groups_to_add.push(group_obj);
}
},
);
}
if !batch.is_empty() {
// Release lock before final persist
_db_lock_holder = None;
@@ -408,6 +495,7 @@ fn expand_series_item(pli: &PlaylistItem, input: &ConfigInput) -> Option<Playlis
None
}
#[allow(clippy::too_many_arguments)]
async fn update_series_info_immediate(
ctx: &PlaylistProcessingContext,
active_provider: &Arc<ActiveProviderManager>,
@@ -416,8 +504,10 @@ async fn update_series_info_immediate(
id: ProviderIdType,
reasons: &ResolveReasonSet,
db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
probe_settings: SeriesProbeSettings,
) -> Result<Option<SeriesStreamProperties>, TuliproxError> {
let fetch_info = reasons.contains(ResolveReason::Info);
let resolve_tmdb = reasons.contains(ResolveReason::Tmdb) || reasons.contains(ResolveReason::Date);
update_series_metadata(
&ctx.config,
@@ -429,7 +519,9 @@ async fn update_series_info_immediate(
Some(&pli.header.title),
false, // save (we batch in caller)
fetch_info,
resolve_tmdb,
reasons.contains(ResolveReason::Probe),
probe_settings,
db_query,
)
.await
@@ -438,12 +530,12 @@ async fn update_series_info_immediate(
fn check_resolve_reasons(
resolve_options: &ResolveOptions,
resolve_tmdb_enabled: bool,
pli: &mut PlaylistItem,
pli: &PlaylistItem,
) -> ResolveReasonSet {
let mut reasons = ResolveReasonSet::new();
let needs_info = check_needs_info(resolve_options, pli, &mut reasons);
check_resolve_tmdb(resolve_options, resolve_tmdb_enabled, pli, needs_info, &mut reasons);
check_needs_info(resolve_options, pli, &mut reasons);
check_resolve_tmdb(resolve_options, resolve_tmdb_enabled, pli, &mut reasons);
if resolve_options.has_flag(ResolveOptionsFlags::Probe) {
check_needs_probe(pli, &mut reasons);
@@ -451,34 +543,36 @@ fn check_resolve_reasons(
reasons
}
fn check_needs_info(resolve_options: &ResolveOptions, pli: &mut PlaylistItem, reasons: &mut ResolveReasonSet) -> bool {
fn check_needs_info(resolve_options: &ResolveOptions, pli: &PlaylistItem, reasons: &mut ResolveReasonSet) {
let needs_info = resolve_options.has_flag(ResolveOptionsFlags::Resolve) && !pli.has_details();
if needs_info {
reasons.set(ResolveReason::Info);
}
needs_info
}
fn check_resolve_tmdb(
resolve_options: &ResolveOptions,
resolve_tmdb_enabled: bool,
pli: &mut PlaylistItem,
needs_info: bool,
pli: &PlaylistItem,
reasons: &mut ResolveReasonSet,
) {
if resolve_tmdb_enabled && resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) && needs_info {
if let Some(StreamProperties::Series(series_stream_props)) = pli.header.additional_properties.as_ref() {
let has_tmdb = series_stream_props.tmdb.is_some();
let has_date = series_stream_props.release_date.is_some();
let title_present = !series_stream_props.name.is_empty() || !pli.header.title.is_empty();
if resolve_tmdb_enabled && resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) {
let (has_tmdb, has_date, title_present) = match pli.header.additional_properties.as_ref() {
Some(StreamProperties::Series(series_stream_props)) => (
series_stream_props.tmdb.is_some(),
series_stream_props.release_date.is_some(),
!series_stream_props.name.is_empty() || !pli.header.title.is_empty(),
),
None => (false, false, !pli.header.title.is_empty()),
_ => return,
};
if title_present && (!has_tmdb || !has_date) {
if !has_tmdb {
reasons.set(ResolveReason::Tmdb);
}
if !has_date {
reasons.set(ResolveReason::Date);
}
if title_present && (!has_tmdb || !has_date) {
if !has_tmdb {
reasons.set(ResolveReason::Tmdb);
}
if !has_date {
reasons.set(ResolveReason::Date);
}
}
}
@@ -502,15 +596,15 @@ fn check_needs_probe(pli: &PlaylistItem, reasons: &mut ResolveReasonSet) {
reasons.set(ResolveReason::Probe);
}
}
/// Updates metadata for a single Series (Info + Episodes Probe) and persists it.
///
/// # Arguments
/// * `save` - If true, persists changes to the input database immediately (Instant strategy).
/// If false, returns the properties so the caller can batch persist them (Bundled strategy).
/// * `fetch_info` - If true, fetches details from Provider API. If false, uses existing/dummy data.
/// * `resolve_tmdb` - If true, resolves missing TMDB/date metadata from available titles.
/// * `db_query` - Optional pre-opened DB handle to avoid re-opening file.
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
#[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::fn_params_excessive_bools)]
pub async fn update_series_metadata(
app_config: &Arc<AppConfig>,
client: &reqwest::Client,
@@ -521,7 +615,9 @@ pub async fn update_series_metadata(
playlist_title: Option<&str>,
save: bool,
fetch_info: bool,
resolve_tmdb: bool,
do_probe: bool,
probe_settings: SeriesProbeSettings,
db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
) -> Result<Option<SeriesStreamProperties>, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
@@ -568,10 +664,9 @@ pub async fn update_series_metadata(
} else {
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Series);
if xtream_path.exists() {
let file_lock = app_config.file_locks.read_lock(&xtream_path).await;
let _file_lock = app_config.file_locks.read_lock(&xtream_path).await;
let xtream_path = xtream_path.clone();
let item = match tokio::task::spawn_blocking(move || {
let _guard = file_lock;
let mut query = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path)?;
query.query_zero_copy(&series_id)
})
@@ -657,7 +752,7 @@ pub async fn update_series_metadata(
let resolve_tmdb_enabled = input.has_flag(ConfigInputFlags::ResolveTmdb);
// 2. Resolve TMDB/Date if missing
if fetch_info
if resolve_tmdb
&& resolve_tmdb_enabled
&& (properties.tmdb.is_none() || properties.release_date.is_none())
&& !properties.name.is_empty()
@@ -721,10 +816,7 @@ pub async fn update_series_metadata(
if let Some(details) = properties.details.as_mut() {
if let Some(episodes) = details.episodes.as_mut() {
let config = app_config.config.load();
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
let user_agent = config.default_user_agent.clone();
let analyze_duration = 10_000_000;
let probe_size = 10_000_000;
let input_url = input.url.as_str();
let input_username = input.username.as_deref().unwrap_or("");
@@ -767,9 +859,9 @@ pub async fn update_series_metadata(
if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url(
&episode_url,
user_agent.as_deref(),
analyze_duration,
probe_size,
ffprobe_timeout,
probe_settings.analyze_duration_micros,
probe_settings.probe_size_bytes,
probe_settings.timeout_secs,
)
.await
{
backend/src/processing/processor/xtream_vod.rs
+67 -15
View File
@@ -7,7 +7,9 @@ use crate::model::{AppConfig, ConfigTarget};
use crate::model::{ConfigInput, ConfigInputFlags};
use crate::processing::processor::playlist::PlaylistProcessingContext;
use crate::processing::processor::{
create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags,
create_resolve_options_function_for_xtream_target, process_foreground_retry_once, ResolveOptions,
ResolveOptionsFlags, FOREGROUND_BATCH_SIZE as BATCH_SIZE, FOREGROUND_MIN_RETRY_DELAY_SECS,
FOREGROUND_RETRY_BATCH_MAX_SIZE as RETRY_BATCH_MAX_SIZE,
};
use crate::ptt::ptt_parse_title;
use crate::repository::get_input_storage_path;
@@ -23,14 +25,12 @@ use shared::model::{
MediaQuality, PlaylistEntry, PlaylistItem, PlaylistItemType, StreamProperties, VideoStreamDetailProperties,
VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamVideoInfo,
};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::{Duration, Instant};
create_resolve_options_function_for_xtream_target!(vod);
const BATCH_SIZE: usize = 200;
#[allow(clippy::too_many_lines)]
pub async fn playlist_resolve_vod(
ctx: &PlaylistProcessingContext,
@@ -162,6 +162,7 @@ async fn process_immediate_vod_info(
let mut _db_lock_holder = None;
let mut batch: Vec<(ProviderIdType, VideoStreamProperties)> = Vec::with_capacity(BATCH_SIZE);
let mut retry_once_ids: HashSet<ProviderIdType> = HashSet::new();
let mut processed_count = 0;
let mut last_log_time = Instant::now();
@@ -267,6 +268,7 @@ async fn process_immediate_vod_info(
Ok(None) => {}
Err(e) => {
error!("Failed to update VOD metadata for {}: {e}", pli.header.title);
retry_once_ids.insert(provider_id.clone());
}
}
}
@@ -278,6 +280,52 @@ async fn process_immediate_vod_info(
}
}
if !retry_once_ids.is_empty() {
let retry_delay_secs = u64::from(resolve_options.resolve_delay).max(FOREGROUND_MIN_RETRY_DELAY_SECS);
process_foreground_retry_once!(
ctx: ctx,
fpl: fpl,
filter: filter,
retry_once_ids: retry_once_ids,
retry_delay_secs: retry_delay_secs,
xtream_path: xtream_path,
db_query_holder: db_query_holder,
db_lock_holder: _db_lock_holder,
batch: batch,
batch_size: BATCH_SIZE,
retry_batch_max_len: RETRY_BATCH_MAX_SIZE,
processed_count: processed_count,
query_error_context: "VOD retry",
reasons: |retry_pli| check_resolve_reasons(&resolve_options, do_probe, resolve_tmdb_enabled, retry_pli),
update: |active_provider, retry_pli, provider_id, reasons, db_query_ref| update_vod_info_immediate(
ctx,
active_provider,
input,
retry_pli,
provider_id,
reasons,
db_query_ref,
),
apply_properties: |retry_pli, updated_props| {
retry_pli.header.additional_properties = Some(StreamProperties::Video(Box::new(updated_props.clone())));
},
persist: |updates| persist_input_vod_info_batch(
&ctx.config,
&storage_path,
XtreamCluster::Video,
&input.name,
updates,
),
on_persist_error: |err| {
error!("persist_input_vod_info_batch failed for VOD retry on input '{}'. Error: {err}", input.name);
},
on_retry_error: |retry_pli, err| {
error!("Foreground retry failed for VOD {}: {err}", retry_pli.header.title);
},
on_after_attempt: |_retry_pli, _retry_succeeded| {},
);
}
// Flush the remaining batch if bundled strategy
if !batch.is_empty() {
// Release lock before final persist
@@ -312,9 +360,9 @@ fn check_resolve_reasons(
// Check if we need to do anything for this item
let mut reasons = ResolveReasonSet::default();
let needs_info = check_needs_info(resolve_options, pli, &mut reasons);
check_needs_info(resolve_options, pli, &mut reasons);
// TMDB check
check_resolve_tmdb(resolve_options, resolve_tmdb_enabled, pli, needs_info, &mut reasons);
check_resolve_tmdb(resolve_options, resolve_tmdb_enabled, pli, &mut reasons);
// Probe check (independent of resolve info fetch; we still probe when A/V details are missing)
if do_probe {
@@ -352,10 +400,9 @@ fn check_resolve_tmdb(
resolve_options: &ResolveOptions,
resolve_tmdb_enabled: bool,
pli: &PlaylistItem,
needs_info: bool,
reasons: &mut ResolveReasonSet,
) {
if resolve_tmdb_enabled && resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) && needs_info {
if resolve_tmdb_enabled && resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) {
if let Some(StreamProperties::Video(video_stream_props)) = pli.header.additional_properties.as_ref() {
let has_tmdb = video_stream_props.tmdb.is_some();
let has_date = video_stream_props.details.as_ref().and_then(|d| d.release_date.as_ref()).is_some();
@@ -415,6 +462,8 @@ async fn update_vod_info_immediate(
db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
) -> Result<Option<VideoStreamProperties>, TuliproxError> {
let fetch_info = reasons.contains(ResolveReason::Info);
let resolve_tmdb =
fetch_info || reasons.contains(ResolveReason::Tmdb) || reasons.contains(ResolveReason::Date);
update_vod_metadata(
&ctx.config,
@@ -426,6 +475,7 @@ async fn update_vod_info_immediate(
Some(&pli.header.title),
false,
fetch_info,
resolve_tmdb,
reasons.contains(ResolveReason::Probe),
db_query,
)
@@ -438,8 +488,9 @@ async fn update_vod_info_immediate(
/// * `save` - If true, persists changes to the input database immediately (Instant strategy).
/// If false, returns the properties so the caller can batch persist them (Bundled strategy).
/// * `fetch_info` - If true, fetches details from Provider API. If false, uses existing/dummy data.
/// * `resolve_tmdb` - If true, resolves missing TMDB/date metadata from available titles.
/// * `db_query` - Optional pre-opened DB handle to avoid re-opening file.
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
#[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::fn_params_excessive_bools)]
pub async fn update_vod_metadata(
app_config: &Arc<AppConfig>,
client: &reqwest::Client,
@@ -450,6 +501,7 @@ pub async fn update_vod_metadata(
playlist_title: Option<&str>,
save: bool,
fetch_info: bool,
resolve_tmdb: bool,
do_probe: bool,
db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
) -> Result<Option<VideoStreamProperties>, TuliproxError> {
@@ -494,10 +546,9 @@ pub async fn update_vod_metadata(
} else {
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Video);
if xtream_path.exists() {
let file_lock = app_config.file_locks.read_lock(&xtream_path).await;
let _file_lock = app_config.file_locks.read_lock(&xtream_path).await;
let xtream_path = xtream_path.clone();
let item = match tokio::task::spawn_blocking(move || {
let _guard = file_lock;
let mut query = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path)?;
query.query_zero_copy(&stream_id)
})
@@ -603,7 +654,7 @@ pub async fn update_vod_metadata(
let missing_tmdb = properties.tmdb.is_none();
let missing_date = properties.details.as_ref().and_then(|d| d.release_date.as_ref()).is_none();
if fetch_info && resolve_tmdb_enabled && (missing_tmdb || missing_date) {
if resolve_tmdb && resolve_tmdb_enabled && (missing_tmdb || missing_date) {
// Try local parsing first
if missing_date && !properties.name.is_empty() {
let meta_parse = ptt_parse_title(&properties.name);
@@ -715,10 +766,11 @@ pub async fn update_vod_metadata(
);
let config = app_config.config.load();
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
let metadata_update = config.metadata_update.clone().unwrap_or_default();
let ffprobe_timeout = metadata_update.ffprobe_timeout.unwrap_or(60);
let user_agent = config.default_user_agent.clone();
let analyze_duration = 10_000_000;
let probe_size = 10_000_000;
let analyze_duration = metadata_update.ffprobe_analyze_duration_micros;
let probe_size = metadata_update.ffprobe_probe_size_bytes;
// Acquire Connection logic
let temp_handle = if active_handle.is_some() {