mirror of
https://github.com/euzu/tuliprox.git
synced 2026-10-20 06:42:20 +02:00
Enhanced Metadata Resolution, Stream Analysis & Improved STRM-Output. Added connection priority management.
**Description:**
This PR introduces a major overhaul of how streams are analyzed and connections are managed. It integrates `ffprobe` for deep stream analysis, adds a background task queue for metadata resolution to prevent UI blocking, introduces a connection priority system, and adds support for Live TV probing.
### 🚀 Key Features
* **FFprobe Integration & Live TV Probing**:
* Automatically probes streams (VOD/Series **and now Live TV**) to determine resolution, codecs (HEVC/AV1), dynamic range (HDR/DV), bit depth, and audio channels.
* **Safe Probing**: Probing strictly respects the provider's `max_connections` limit.
* Docker images now include static `ffmpeg/ffprobe` binaries.
* **Smart Connection Management (Priority System)**:
* Introduced a priority system for connections to manage provider limits intelligently.
* **Preemption**: Users with higher priority can preempt (kick) lower priority connections when provider slots are full.
* **Background Queue**: Metadata resolution and stream analysis run as background tasks (Priority 0). They only run when connection slots are idle and yield immediately if a real user needs to stream.
* **Enhanced Metadata Resolution**:
* **Fallback Logic**: If a provider is missing the TMDB ID or Release Date, the system resolves them via the TMDB API.
* **Smart Title Cleaning**: Implemented a centralized cleaner to strip common IPTV prefixes before attempting metadata matches or generating filenames, significantly improving match rates.
* **Improved Output**:
* **Quality Tags**: Filenames in STRM files now include detailed tags (e.g., `Movie - [4K HEVC HDR].strm`).
* **Flat Grouping**: `flat: true` now correctly groups multi-version movies into single folders based on TMDB ID.
* **Global Series Date**: Ensures consistent series folder naming even if episodes have different years.
This commit is contained in:
@@ -103,18 +103,19 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &Arc<str>, group_nam
|
||||
item_type: PlaylistItemType::LocalSeries,
|
||||
category_id: 0,
|
||||
input_name: Arc::clone(input_name),
|
||||
additional_properties: Some(StreamProperties::Episode(EpisodeStreamProperties {
|
||||
additional_properties: Some(StreamProperties::Episode(Box::new(EpisodeStreamProperties {
|
||||
episode_id: episode.id,
|
||||
episode: episode.episode_num,
|
||||
season: episode.season,
|
||||
added: Some(episode.added.clone()),
|
||||
release_date: Some(episode.release_date.clone()),
|
||||
series_release_date: None,
|
||||
tmdb: episode.tmdb,
|
||||
movie_image: logo,
|
||||
container_extension: container_extension.intern(),
|
||||
audio: None,
|
||||
video: None,
|
||||
})),
|
||||
}))),
|
||||
..Default::default()
|
||||
}
|
||||
});
|
||||
@@ -287,6 +288,7 @@ pub fn metadata_cache_entry_to_xtream_series_info(
|
||||
direct_source: episode.file_path.clone().into(),
|
||||
tmdb: tmdb_id,
|
||||
release_date: episode_release_date.clone().into(),
|
||||
series_release_date: None,
|
||||
plot: episode.plot.clone().map(Into::into),
|
||||
crew: Some(Arc::clone(&actor_names)),
|
||||
duration_secs: episode.runtime.map_or(0, |r| r * 60),
|
||||
|
||||
@@ -1,29 +1,76 @@
|
||||
pub mod playlist;
|
||||
mod playlist;
|
||||
mod xtream;
|
||||
// mod affix;
|
||||
mod xtream_vod;
|
||||
mod xtream_series;
|
||||
pub mod epg;
|
||||
mod epg;
|
||||
mod sort;
|
||||
pub mod trakt;
|
||||
mod trakt;
|
||||
mod library;
|
||||
mod stream_probe;
|
||||
pub use self::playlist::*;
|
||||
pub use self::epg::*;
|
||||
pub use self::xtream::*;
|
||||
pub use self::xtream_vod::*;
|
||||
pub use self::xtream_series::*;
|
||||
pub use self::stream_probe::*;
|
||||
use shared::create_bit_set;
|
||||
|
||||
create_bit_set!(u16, ResolveOptionsFlags, Resolve, TmdbMissing, Probe, Background);
|
||||
|
||||
pub struct ResolveOptions {
|
||||
pub flags: ResolveOptionsFlagsSet,
|
||||
pub resolve_delay: u16,
|
||||
}
|
||||
|
||||
impl Default for ResolveOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
flags: ResolveOptionsFlagsSet::from_variants(&[ResolveOptionsFlags::Background]),
|
||||
resolve_delay: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// fn get_resolve_<cluster>_options(target: &ConfigTarget, fpl: &FetchedPlaylist) -> (bool, u16)
|
||||
// fn get_resolve_<cluster>_options(target: &ConfigTarget, fpl: &FetchedPlaylist) -> bool
|
||||
//
|
||||
#[macro_export]
|
||||
macro_rules! create_resolve_options_function_for_xtream_target {
|
||||
($cluster:ident) => {
|
||||
paste::paste! {
|
||||
fn [<get_resolve_ $cluster _options>](target: &ConfigTarget, fpl: &FetchedPlaylist) -> (bool, u16) {
|
||||
fn [<get_resolve_ $cluster _options>](target: &ConfigTarget, fpl: &FetchedPlaylist) -> $crate::processing::processor::ResolveOptions {
|
||||
// Get input options
|
||||
let (resolve_tmdb_missing, probe_requested) = fpl.input.options.as_ref().map_or_else(||(false, false), |o| (o.resolve_tmdb, o.probe_stream));
|
||||
match target.get_xtream_output() {
|
||||
Some(xtream_output) => (xtream_output.[<resolve_ $cluster>] && fpl.input.input_type == InputType::Xtream,
|
||||
xtream_output.[<resolve_ $cluster _delay>]),
|
||||
None => (false, 0)
|
||||
Some(xtream_output) => {
|
||||
let mut flags = $crate::processing::processor::ResolveOptionsFlagsSet::new();
|
||||
// Map config flags to resolve options flags
|
||||
if xtream_output.flags.contains($crate::model::XtreamTargetFlags::[<Resolve $cluster:camel>]) && fpl.input.input_type.is_xtream() {
|
||||
flags.add($crate::processing::processor::ResolveOptionsFlags::Resolve);
|
||||
}
|
||||
if resolve_tmdb_missing {
|
||||
flags.add($crate::processing::processor::ResolveOptionsFlags::TmdbMissing);
|
||||
}
|
||||
if probe_requested
|
||||
&& fpl.input.input_type.is_xtream()
|
||||
&& xtream_output.flags.contains($crate::model::XtreamTargetFlags::[<Probe $cluster:camel>]) {
|
||||
flags.add($crate::processing::processor::ResolveOptionsFlags::Probe);
|
||||
}
|
||||
if xtream_output.flags.contains($crate::model::XtreamTargetFlags::ResolveBackground) {
|
||||
flags.add($crate::processing::processor::ResolveOptionsFlags::Background);
|
||||
}
|
||||
|
||||
$crate::processing::processor::ResolveOptions {
|
||||
flags,
|
||||
resolve_delay: xtream_output.resolve_delay,
|
||||
}
|
||||
},
|
||||
None => $crate::processing::processor::ResolveOptions::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
use create_resolve_options_function_for_xtream_target;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::model::{AppConfig, ConfigFavourites, ConfigInput, ConfigRename, ReverseProxyDisabledHeaderConfig, TVGuide};
|
||||
use crate::model::{AppConfig, ConfigFavourites, ConfigInput, ConfigRename, MessageContent, ReverseProxyDisabledHeaderConfig, TVGuide};
|
||||
use crate::utils::{log_memory_snapshot, m3u};
|
||||
use crate::utils::xtream;
|
||||
use crate::utils::{epg, StepMeasureCallback};
|
||||
@@ -8,15 +8,13 @@ use std::sync::{Arc, Weak};
|
||||
use tokio::sync::{Mutex, OwnedRwLockWriteGuard, RwLock};
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
use crate::api::model::{EventManager, EventMessage, PlaylistStorageState, UpdateGuard};
|
||||
use crate::messaging::send_message;
|
||||
use crate::model::messaging::MessageContent;
|
||||
use crate::api::model::{ActiveProviderManager, EventManager, EventMessage, PlaylistStorageState, UpdateGuard, MetadataUpdateManager, ProviderIdType, ResolveReasonSet, ResolveReason, UpdateTask};
|
||||
use crate::messaging::{send_message};
|
||||
|
||||
|
||||
use crate::model::FetchedPlaylist;
|
||||
use crate::model::Mapping;
|
||||
use crate::model::{ConfigTarget, ProcessTargets};
|
||||
use crate::model::{InputStats, PlaylistStats, SourceStats, TargetStats};
|
||||
use crate::model::{ConfigTarget, ProcessTargets, TargetOutput, XtreamTargetFlags};
|
||||
use crate::processing::input_cache;
|
||||
use crate::processing::input_cache::ClusterState;
|
||||
use crate::processing::parser::xmltv::flatten_tvguide;
|
||||
@@ -39,13 +37,15 @@ use shared::error::{get_errors_notify_message, notify_err, TuliproxError};
|
||||
use shared::foundation::{get_field_value, set_field_value, ValueAccessor, ValueProvider};
|
||||
use shared::foundation::Filter;
|
||||
use shared::model::xtream_const::XTREAM_CLUSTER;
|
||||
use shared::model::UUIDType;
|
||||
use shared::model::{InputStats, PlaylistStats, SourceStats, TargetStats, UUIDType};
|
||||
use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField,
|
||||
PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState,
|
||||
ProcessingOrder, XtreamCluster};
|
||||
ProcessingOrder, StreamProperties, XtreamCluster};
|
||||
use shared::utils::{create_alias_uuid, default_as_default, interner_gc, Internable};
|
||||
use std::time::Instant;
|
||||
|
||||
const DEFAULT_PROBE_STREAM_DELAY: u16 = 50;
|
||||
|
||||
fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool {
|
||||
let provider = ValueProvider { pli, match_as_ascii };
|
||||
filter.filter(&provider)
|
||||
@@ -276,7 +276,30 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo
|
||||
(!user_targets.enabled && target.enabled) || (user_targets.enabled && user_targets.has_target(target.id))
|
||||
}
|
||||
|
||||
async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc<AppConfig>, input: &ConfigInput) -> (Vec<PlaylistGroup>, Vec<TuliproxError>, bool, bool) {
|
||||
struct PlaylistDownloadResult {
|
||||
pub downloaded_playlist:Vec<PlaylistGroup>,
|
||||
pub download_err: Vec<TuliproxError>,
|
||||
pub was_cached: bool,
|
||||
pub persisted: bool,
|
||||
}
|
||||
|
||||
impl PlaylistDownloadResult {
|
||||
pub fn new(
|
||||
downloaded_playlist:Vec<PlaylistGroup>,
|
||||
download_err: Vec<TuliproxError>,
|
||||
was_cached: bool,
|
||||
persisted: bool) -> Self {
|
||||
|
||||
Self {
|
||||
downloaded_playlist,
|
||||
download_err,
|
||||
was_cached,
|
||||
persisted,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc<AppConfig>, input: &ConfigInput) -> PlaylistDownloadResult {
|
||||
let config = &*app_config.config.load();
|
||||
let working_dir = &config.working_dir;
|
||||
|
||||
@@ -315,7 +338,7 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc
|
||||
};
|
||||
|
||||
if fully_cached {
|
||||
return (vec![], vec![], true, false);
|
||||
return PlaylistDownloadResult::new(vec![], vec![], true, false);
|
||||
}
|
||||
|
||||
let (playlist, errors, persisted) = match input.input_type {
|
||||
@@ -365,7 +388,7 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc
|
||||
}
|
||||
}
|
||||
|
||||
(playlist, errors, false, persisted)
|
||||
PlaylistDownloadResult::new(playlist, errors, false, persisted)
|
||||
}
|
||||
|
||||
async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (Vec<InputStats>, Vec<TargetStats>, Vec<TuliproxError>) {
|
||||
@@ -489,7 +512,7 @@ async fn download_input(ctx: &PlaylistProcessingContext, input: &Arc<ConfigInput
|
||||
// Coordination Logic
|
||||
let need_download = !ctx.is_input_downloaded(&input.name).await;
|
||||
|
||||
let (downloaded_playlist, download_err, was_cached, persisted) = if need_download {
|
||||
let playlist_download_result= if need_download {
|
||||
// Acquire named lock to prevent thundering herd on same input
|
||||
let _input_lock = ctx.get_input_lock(&input.name).await;
|
||||
// Check again after lock
|
||||
@@ -497,7 +520,7 @@ async fn download_input(ctx: &PlaylistProcessingContext, input: &Arc<ConfigInput
|
||||
|
||||
if already_processed {
|
||||
// Use empty results, will load from disk below
|
||||
(vec![], vec![], true, false)
|
||||
PlaylistDownloadResult::new(vec![], vec![], true, false)
|
||||
} else {
|
||||
let res = playlist_download_from_input(&ctx.client, &ctx.config, input).await;
|
||||
// Mark as processed if NO critical errors?
|
||||
@@ -507,20 +530,20 @@ async fn download_input(ctx: &PlaylistProcessingContext, input: &Arc<ConfigInput
|
||||
res
|
||||
}
|
||||
} else {
|
||||
(vec![], vec![], true, false)
|
||||
PlaylistDownloadResult::new(vec![], vec![], true, false)
|
||||
};
|
||||
|
||||
let (playlist, error) = if was_cached || persisted {
|
||||
let (playlist, error) = if playlist_download_result.was_cached || playlist_download_result.persisted {
|
||||
match load_input_playlist(ctx, input, None).await {
|
||||
Ok(pl_source) => (pl_source, None),
|
||||
Err(e) => (MemoryPlaylistSource::default().boxed(), Some(e)),
|
||||
}
|
||||
} else {
|
||||
debug!("Persisting input '{}' playlist", input.name);
|
||||
let (pl, err) = persist_input_playlist(&ctx.config, input, downloaded_playlist).await;
|
||||
let (pl, err) = persist_input_playlist(&ctx.config, input, playlist_download_result.downloaded_playlist).await;
|
||||
(MemoryPlaylistSource::new(pl).boxed(), err)
|
||||
};
|
||||
(download_err, playlist, error)
|
||||
(playlist_download_result.download_err, playlist, error)
|
||||
}
|
||||
|
||||
fn create_broadcast_callback(event_manager: Option<&Arc<EventManager>>) -> StepMeasureCallback {
|
||||
@@ -562,9 +585,13 @@ pub struct PlaylistProcessingContext {
|
||||
pub disabled_headers: Option<ReverseProxyDisabledHeaderConfig>,
|
||||
|
||||
// Coordination
|
||||
processed_inputs: Arc<Mutex<HashSet<Arc<str>>>>,
|
||||
pub processed_inputs: Arc<Mutex<HashSet<Arc<str>>>>,
|
||||
#[allow(clippy::type_complexity)]
|
||||
input_locks: Arc<Mutex<HashMap<Arc<str>, Weak<RwLock<()>>>>>,
|
||||
pub input_locks: Arc<Mutex<HashMap<Arc<str>, Weak<RwLock<()>>>>>,
|
||||
|
||||
// New field for STRM probes & background updates
|
||||
pub provider_manager: Option<Arc<ActiveProviderManager>>,
|
||||
pub metadata_manager: Option<Arc<MetadataUpdateManager>>,
|
||||
}
|
||||
|
||||
impl PlaylistProcessingContext {
|
||||
@@ -754,9 +781,7 @@ async fn process_playlist_for_target(ctx: &PlaylistProcessingContext,
|
||||
let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates, consume_input_source);
|
||||
log_memory_snapshot(format!("target '{}' input '{}' after_pipe", target.name, provider_fpl.input.name).as_str());
|
||||
processed_fpl.sort_by_provider_ordinal();
|
||||
playlist_resolve_series(&ctx.config, &ctx.client, target, errors, &pipe, provider_fpl, &mut processed_fpl).await;
|
||||
log_memory_snapshot(format!("target '{}' input '{}' after_series_resolve", target.name, provider_fpl.input.name).as_str());
|
||||
playlist_resolve_vod(&ctx.config, &ctx.client, target, errors, provider_fpl, &mut processed_fpl).await;
|
||||
playlist_resolve(ctx, target, errors, &pipe, provider_fpl, &mut processed_fpl).await;
|
||||
log_memory_snapshot(format!("target '{}' input '{}' after_vod_resolve", target.name, provider_fpl.input.name).as_str());
|
||||
// stats
|
||||
let input_entry_name = processed_fpl.input.name.clone();
|
||||
@@ -818,6 +843,203 @@ async fn process_playlist_for_target(ctx: &PlaylistProcessingContext,
|
||||
}
|
||||
}
|
||||
|
||||
async fn playlist_resolve(ctx: &PlaylistProcessingContext,
|
||||
target: &ConfigTarget,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
pipe: &ProcessingPipe,
|
||||
provider_fpl: &mut FetchedPlaylist<'_>,
|
||||
processed_fpl: &mut FetchedPlaylist<'_>,
|
||||
) {
|
||||
|
||||
playlist_resolve_series(ctx, target, errors, pipe, provider_fpl, processed_fpl).await;
|
||||
playlist_resolve_vod(ctx, target, errors, provider_fpl, processed_fpl).await;
|
||||
playlist_probe(ctx, target, processed_fpl).await;
|
||||
}
|
||||
|
||||
fn is_xtream_only_output_target(target: &ConfigTarget) -> bool {
|
||||
target.output.len() == 1 && matches!(target.output.first(), Some(TargetOutput::Xtream(_)))
|
||||
}
|
||||
|
||||
fn get_xtream_cluster_probe_flag(target: &ConfigTarget, cluster: XtreamCluster) -> Option<bool> {
|
||||
target.get_xtream_output().map(|xtream_output| match cluster {
|
||||
XtreamCluster::Live => xtream_output.flags.contains(XtreamTargetFlags::ProbeLive),
|
||||
XtreamCluster::Video => xtream_output.flags.contains(XtreamTargetFlags::ProbeVod),
|
||||
XtreamCluster::Series => xtream_output.flags.contains(XtreamTargetFlags::ProbeSeries),
|
||||
})
|
||||
}
|
||||
|
||||
fn is_cluster_probe_enabled_for_target(target: &ConfigTarget, cluster: XtreamCluster) -> bool {
|
||||
// Probe flags are only authoritative when the target is exactly one xtream output.
|
||||
// For mixed outputs (xtream + m3u/strm/...), probing remains enabled because non-xtream outputs have no probe flags.
|
||||
if !is_xtream_only_output_target(target) {
|
||||
return true;
|
||||
}
|
||||
|
||||
get_xtream_cluster_probe_flag(target, cluster).unwrap_or(true)
|
||||
}
|
||||
|
||||
fn is_probe_supported_item_type(item_type: PlaylistItemType) -> bool {
|
||||
matches!(
|
||||
item_type,
|
||||
PlaylistItemType::Live // we skip other live streams because hls and dash have multiple resolutions
|
||||
| PlaylistItemType::Video
|
||||
| PlaylistItemType::LocalVideo
|
||||
| PlaylistItemType::Series
|
||||
| PlaylistItemType::LocalSeries
|
||||
)
|
||||
}
|
||||
|
||||
fn has_probe_details(item: &PlaylistItem) -> bool {
|
||||
match item.header.additional_properties.as_ref() {
|
||||
Some(StreamProperties::Video(v)) => v
|
||||
.details
|
||||
.as_ref()
|
||||
.is_some_and(|d| d.video.is_some() && d.audio.is_some()),
|
||||
Some(StreamProperties::Live(l)) => l.video.is_some() && l.audio.is_some(),
|
||||
Some(StreamProperties::Episode(e)) => e.video.is_some() && e.audio.is_some(),
|
||||
Some(StreamProperties::Series(_)) | None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_live_probe_interval_settings(target: &ConfigTarget, input_type: InputType) -> Option<(u16, u64)> {
|
||||
if !matches!(input_type, InputType::Xtream | InputType::XtreamBatch) {
|
||||
return None;
|
||||
}
|
||||
target
|
||||
.get_xtream_output()
|
||||
.map(|xtream_output| (xtream_output.resolve_delay, u64::from(xtream_output.probe_live_interval_hours) * 3600))
|
||||
}
|
||||
|
||||
fn needs_live_probe(item: &PlaylistItem, cutoff_ts: i64) -> bool {
|
||||
match item.header.additional_properties.as_ref() {
|
||||
Some(StreamProperties::Live(props)) => {
|
||||
if let Some(last_ts) = props.last_probed_timestamp {
|
||||
last_ts < cutoff_ts
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_id_from_item(item: &PlaylistItem) -> Option<ProviderIdType> {
|
||||
if let Ok(id) = item.header.id.parse::<u32>() {
|
||||
if id == 0 {
|
||||
return None;
|
||||
}
|
||||
return Some(ProviderIdType::Id(id));
|
||||
}
|
||||
|
||||
let raw = item.header.id.trim();
|
||||
if raw.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ProviderIdType::from(raw))
|
||||
}
|
||||
}
|
||||
|
||||
async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget, fpl: &mut FetchedPlaylist<'_>) {
|
||||
let Some(mgr) = ctx.metadata_manager.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let Some(opts) = fpl.input.options.as_ref() else {
|
||||
return;
|
||||
};
|
||||
if !opts.probe_stream {
|
||||
return;
|
||||
}
|
||||
if !ctx.config.is_ffprobe_enabled().await {
|
||||
return;
|
||||
}
|
||||
|
||||
let input_name = fpl.input.name.clone();
|
||||
let input_type = fpl.input.input_type;
|
||||
let xtream_only_output = is_xtream_only_output_target(target);
|
||||
let live_probe_settings = get_live_probe_interval_settings(target, input_type).map(|(delay, interval_secs)| {
|
||||
let interval_signed = i64::try_from(interval_secs).unwrap_or(i64::MAX);
|
||||
let cutoff_ts = chrono::Utc::now().timestamp().saturating_sub(interval_signed);
|
||||
(delay, interval_secs, cutoff_ts)
|
||||
});
|
||||
|
||||
let mut queued_probe_keys: HashSet<(Arc<str>, String)> = HashSet::new();
|
||||
let mut queued_live_keys: HashSet<ProviderIdType> = HashSet::new();
|
||||
let mut queued_live_count = 0usize;
|
||||
let mut queued_stream_count = 0usize;
|
||||
|
||||
for item in fpl.items() {
|
||||
if !is_cluster_probe_enabled_for_target(target, item.header.xtream_cluster) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !is_probe_supported_item_type(item.header.item_type) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if item.header.item_type.is_live() {
|
||||
if let Some((resolve_delay, interval_secs, cutoff_ts)) = live_probe_settings {
|
||||
if needs_live_probe(&item, cutoff_ts) {
|
||||
if let Some(provider_id) = provider_id_from_item(&item) {
|
||||
if queued_live_keys.insert(provider_id.clone()) {
|
||||
let task = UpdateTask::ProbeLive {
|
||||
id: provider_id,
|
||||
reason: ResolveReasonSet::from_variants(&[ResolveReason::Probe]),
|
||||
delay: resolve_delay,
|
||||
interval: interval_secs,
|
||||
};
|
||||
mgr.queue_task_background(input_name.clone(), task);
|
||||
queued_live_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// For xtream-only targets, dedicated xtream resolve/probe processors handle non-live paths.
|
||||
if xtream_only_output && matches!(input_type, InputType::Xtream | InputType::XtreamBatch) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if has_probe_details(&item) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// For M3U, ID is provider id; for Library, ID is UUID.
|
||||
let unique_id = if input_type == InputType::Library {
|
||||
item.header.uuid.to_valid_uuid()
|
||||
} else {
|
||||
item.header.id.to_string()
|
||||
};
|
||||
let probe_scope = if item.header.input_name.is_empty() {
|
||||
input_name.clone()
|
||||
} else {
|
||||
item.header.input_name.clone()
|
||||
};
|
||||
|
||||
if !queued_probe_keys.insert((probe_scope.clone(), unique_id.clone())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let task = UpdateTask::ProbeStream {
|
||||
probe_scope,
|
||||
unique_id,
|
||||
url: item.header.url.to_string(),
|
||||
item_type: item.header.item_type,
|
||||
reason: ResolveReasonSet::from_variants(&[ResolveReason::MissingDetails]),
|
||||
delay: DEFAULT_PROBE_STREAM_DELAY,
|
||||
};
|
||||
mgr.queue_task_background(input_name.clone(), task);
|
||||
queued_stream_count += 1;
|
||||
}
|
||||
|
||||
if queued_live_count > 0 || queued_stream_count > 0 {
|
||||
info!(
|
||||
"Queued probe tasks for input {input_name} (live_interval={queued_live_count}, generic={queued_stream_count})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn process_favourites(playlist: &mut Vec<PlaylistGroup>, favourites_cfg: Option<&[ConfigFavourites]>) {
|
||||
if let Some(favourites) = favourites_cfg {
|
||||
let mut fav_groups: IndexMap<CategoryKey, Vec<PlaylistItem>> = IndexMap::new();
|
||||
@@ -895,11 +1117,13 @@ async fn process_watch(app_config: &Arc<AppConfig>, client: &reqwest::Client, ta
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn exec_processing(client: &reqwest::Client, app_config: Arc<AppConfig>, targets: Arc<ProcessTargets>,
|
||||
event_manager: Option<Arc<EventManager>>, playlist_state: Option<Arc<PlaylistStorageState>>,
|
||||
update_guard: Option<UpdateGuard>,
|
||||
disabled_headers: Option<ReverseProxyDisabledHeaderConfig>) {
|
||||
let _guard = if let Some(guard) = update_guard {
|
||||
update_guard: Option<UpdateGuard>, disabled_headers: Option<ReverseProxyDisabledHeaderConfig>,
|
||||
provider_manager: Option<Arc<ActiveProviderManager>>,
|
||||
metadata_manager: Option<Arc<MetadataUpdateManager>>) {
|
||||
let _guard = if let Some(guard) = update_guard {
|
||||
if let Some(permit) = guard.try_playlist() {
|
||||
Some(permit)
|
||||
} else {
|
||||
@@ -925,6 +1149,8 @@ pub async fn exec_processing(client: &reqwest::Client, app_config: Arc<AppConfig
|
||||
processed_inputs: Arc::new(Mutex::new(HashSet::new())),
|
||||
input_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||
disabled_headers,
|
||||
provider_manager,
|
||||
metadata_manager,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
@@ -968,19 +1194,67 @@ pub async fn exec_processing(client: &reqwest::Client, app_config: Arc<AppConfig
|
||||
info!("{update_finished_message}");
|
||||
}
|
||||
|
||||
// #[cfg(test)]
|
||||
// mod tests {
|
||||
// #[test]
|
||||
// fn test_jaro_winkeler() {
|
||||
// let data = [("yessport5", "heyessport5gold"), ("yessport5", "heyesport5gold")];
|
||||
//
|
||||
// data.iter().for_each(|(first, second)|
|
||||
// println!("jaro_winkler {} = {} => {}", first, second, strsim::jaro_winkler(first, second)));
|
||||
// // println!("jaro {}", strsim::jaro(data.0, data.1));
|
||||
// // println!("levenhstein {}", strsim::levenshtein(data.0, data.1));
|
||||
// // println!("damerau_levenshtein {:?}", strsim::damerau_levenshtein(data.0, data.1));
|
||||
// // println!("osa distance {:?}", strsim::osa_distance(data.0, data.1));
|
||||
// // println!("sorensen dice {:?}", strsim::sorensen_dice(data.0, data.1));
|
||||
// }
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use shared::utils::Internable;
|
||||
|
||||
// }
|
||||
fn item_with_props(props: StreamProperties) -> PlaylistItem {
|
||||
let header = shared::model::PlaylistItemHeader {
|
||||
additional_properties: Some(props),
|
||||
..Default::default()
|
||||
};
|
||||
PlaylistItem { header }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_probe_details_requires_video_and_audio_for_video() {
|
||||
let video = shared::model::VideoStreamProperties {
|
||||
details: Some(shared::model::VideoStreamDetailProperties {
|
||||
video: Some("{\"codec_name\":\"h264\"}".intern()),
|
||||
audio: None,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let item_missing_audio = item_with_props(StreamProperties::Video(Box::new(video)));
|
||||
assert!(!has_probe_details(&item_missing_audio));
|
||||
|
||||
let video_complete = shared::model::VideoStreamProperties {
|
||||
details: Some(shared::model::VideoStreamDetailProperties {
|
||||
video: Some("{\"codec_name\":\"h264\"}".intern()),
|
||||
audio: Some("{\"codec_name\":\"aac\"}".intern()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let item_complete = item_with_props(StreamProperties::Video(Box::new(video_complete)));
|
||||
assert!(has_probe_details(&item_complete));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_probe_details_requires_video_and_audio_for_live() {
|
||||
let live_missing_audio = shared::model::LiveStreamProperties {
|
||||
video: Some("{\"codec_name\":\"h264\"}".intern()),
|
||||
audio: None,
|
||||
..Default::default()
|
||||
};
|
||||
let item_missing_audio = item_with_props(StreamProperties::Live(Box::new(live_missing_audio)));
|
||||
assert!(!has_probe_details(&item_missing_audio));
|
||||
|
||||
let live_complete = shared::model::LiveStreamProperties {
|
||||
video: Some("{\"codec_name\":\"h264\"}".intern()),
|
||||
audio: Some("{\"codec_name\":\"aac\"}".intern()),
|
||||
..Default::default()
|
||||
};
|
||||
let item_complete = item_with_props(StreamProperties::Live(Box::new(live_complete)));
|
||||
assert!(has_probe_details(&item_complete));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_probe_details_is_false_for_series() {
|
||||
let series = shared::model::SeriesStreamProperties::default();
|
||||
let item = item_with_props(StreamProperties::Series(Box::new(series)));
|
||||
assert!(!has_probe_details(&item));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
use crate::api::model::ActiveProviderManager;
|
||||
use crate::model::ConfigInput;
|
||||
use crate::model::{AppConfig};
|
||||
use crate::repository::{get_input_m3u_playlist_file_path, get_input_storage_path, get_input_local_library_playlist_file_path, xtream_get_file_path, BPlusTreeUpdate};
|
||||
use crate::utils::{debug_if_enabled, ffmpeg};
|
||||
use log::{info, warn};
|
||||
use shared::error::TuliproxError;
|
||||
use shared::model::{EpisodeStreamProperties, InputType, PlaylistItemType, StreamProperties, VideoStreamDetailProperties, VideoStreamProperties, LiveStreamProperties, M3uPlaylistItem, XtreamCluster, XtreamPlaylistItem};
|
||||
use std::sync::Arc;
|
||||
use shared::model::UUIDType;
|
||||
|
||||
enum ProbeStorageKind {
|
||||
M3u,
|
||||
Library,
|
||||
Xtream,
|
||||
}
|
||||
|
||||
fn requires_provider_connection_for_generic_probe(input_type: InputType) -> bool {
|
||||
!matches!(input_type, InputType::Library)
|
||||
}
|
||||
|
||||
/// Updates metadata (Probing) for a stream URL (M3U, Xtream, Library) and persists it.
|
||||
/// - `unique_id`: For M3U this is the `provider_id` (String). For Library this is the `UUID` string.
|
||||
/// For Xtream this is the numeric provider id as string.
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub async fn update_generic_stream_metadata(
|
||||
app_config: &Arc<AppConfig>,
|
||||
input: &ConfigInput,
|
||||
unique_id: &str,
|
||||
stream_url: &str,
|
||||
item_type: PlaylistItemType,
|
||||
active_provider: &Arc<ActiveProviderManager>,
|
||||
active_handle: Option<&crate::api::model::ProviderHandle>,
|
||||
) -> Result<(), TuliproxError> {
|
||||
let working_dir = &app_config.config.load().working_dir;
|
||||
|
||||
// Check if probing is enabled globally
|
||||
let ffprobe_enabled = app_config.is_ffprobe_enabled().await;
|
||||
if !ffprobe_enabled {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Determine storage file path based on input type
|
||||
let storage_path = get_input_storage_path(&input.name, working_dir).await
|
||||
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
|
||||
|
||||
let (db_path, storage_kind) = match input.input_type {
|
||||
InputType::M3u | InputType::M3uBatch => (
|
||||
get_input_m3u_playlist_file_path(&storage_path, &input.name),
|
||||
ProbeStorageKind::M3u,
|
||||
),
|
||||
InputType::Library => (
|
||||
get_input_local_library_playlist_file_path(&storage_path, &input.name),
|
||||
ProbeStorageKind::Library,
|
||||
),
|
||||
InputType::Xtream | InputType::XtreamBatch => {
|
||||
let cluster = if item_type.is_live() {
|
||||
XtreamCluster::Live
|
||||
} else if matches!(item_type, PlaylistItemType::Video | PlaylistItemType::LocalVideo) {
|
||||
XtreamCluster::Video
|
||||
} else if matches!(item_type, PlaylistItemType::Series | PlaylistItemType::LocalSeries) {
|
||||
XtreamCluster::Series
|
||||
} else {
|
||||
// Generic probing currently supports live/video/series payload shapes.
|
||||
return Ok(());
|
||||
};
|
||||
(
|
||||
xtream_get_file_path(&storage_path, cluster),
|
||||
ProbeStorageKind::Xtream,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
if !db_path.exists() {
|
||||
return Err(shared::error::info_err!("Playlist DB file not found for input {}: {}", input.name, db_path.display()));
|
||||
}
|
||||
|
||||
let needs_provider_connection = requires_provider_connection_for_generic_probe(input.input_type);
|
||||
|
||||
let acquired_handle = if !needs_provider_connection || active_handle.is_some() {
|
||||
None
|
||||
} else {
|
||||
active_provider.acquire_connection_for_probe(&input.name).await
|
||||
};
|
||||
|
||||
if needs_provider_connection && active_handle.is_none() && acquired_handle.is_none() {
|
||||
warn!("Skipping probe for generic stream {unique_id} due to connection limits");
|
||||
return Err(shared::error::info_err!("No connection available"));
|
||||
}
|
||||
|
||||
let probe_url = stream_url.to_string();
|
||||
let config = app_config.config.load();
|
||||
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
|
||||
let user_agent = config.default_user_agent.clone();
|
||||
let analyze_duration = 10_000_000;
|
||||
let probe_size = 10_000_000;
|
||||
|
||||
debug_if_enabled!("Probing Generic Stream '{unique_id}'");
|
||||
|
||||
let probe_data = ffmpeg::probe_url(
|
||||
&probe_url,
|
||||
user_agent.as_deref(),
|
||||
analyze_duration,
|
||||
probe_size,
|
||||
ffprobe_timeout,
|
||||
).await;
|
||||
|
||||
if let Some(handle) = acquired_handle {
|
||||
active_provider.release_handle(&handle).await;
|
||||
}
|
||||
|
||||
let Some((_quality, raw_video, raw_audio)) = probe_data else {
|
||||
warn!("Probe failed or timed out for generic stream: {unique_id}");
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
// Acquire lock and open a tree for update
|
||||
let _file_lock = app_config.file_locks.write_lock(&db_path).await;
|
||||
|
||||
// Update the record in BPlusTree
|
||||
match storage_kind {
|
||||
ProbeStorageKind::M3u => {
|
||||
let key: Arc<str> = unique_id.into();
|
||||
let mut tree_update = BPlusTreeUpdate::<Arc<str>, M3uPlaylistItem>::try_new(&db_path)
|
||||
.map_err(|e| shared::error::info_err!("Failed to open M3U tree update: {e}"))?;
|
||||
|
||||
if let Some(mut item) = tree_update
|
||||
.query(&key)
|
||||
.map_err(|e| shared::error::info_err!("Tree query error: {e}"))?
|
||||
{
|
||||
update_properties(
|
||||
&mut item.additional_properties,
|
||||
item_type,
|
||||
&item.name,
|
||||
item.virtual_id,
|
||||
raw_video,
|
||||
raw_audio,
|
||||
);
|
||||
tree_update
|
||||
.update(&key, item)
|
||||
.map_err(|e| shared::error::info_err!("Tree update error: {e}"))?;
|
||||
info!("Successfully updated M3U metadata for: {unique_id}");
|
||||
} else {
|
||||
warn!("Item not found in M3U DB: {unique_id}");
|
||||
}
|
||||
}
|
||||
ProbeStorageKind::Library => {
|
||||
let mut tree_update = BPlusTreeUpdate::<UUIDType, XtreamPlaylistItem>::try_new(&db_path)
|
||||
.map_err(|e| shared::error::info_err!("Failed to open Library tree update: {e}"))?;
|
||||
let uuid = UUIDType::from_valid_uuid(unique_id);
|
||||
|
||||
if let Some(mut item) = tree_update
|
||||
.query(&uuid)
|
||||
.map_err(|e| shared::error::info_err!("Tree query error: {e}"))?
|
||||
{
|
||||
update_properties(
|
||||
&mut item.additional_properties,
|
||||
item_type,
|
||||
&item.name,
|
||||
item.virtual_id,
|
||||
raw_video,
|
||||
raw_audio,
|
||||
);
|
||||
tree_update
|
||||
.update(&uuid, item)
|
||||
.map_err(|e| shared::error::info_err!("Tree update error: {e}"))?;
|
||||
info!("Successfully updated Library metadata for: {unique_id}");
|
||||
} else {
|
||||
warn!("Item not found in Library DB: {unique_id}");
|
||||
}
|
||||
}
|
||||
ProbeStorageKind::Xtream => {
|
||||
let Ok(provider_id) = unique_id.parse::<u32>() else {
|
||||
warn!("Skipping xtream generic probe update with non-numeric id: {unique_id}");
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut tree_update = BPlusTreeUpdate::<u32, XtreamPlaylistItem>::try_new(&db_path)
|
||||
.map_err(|e| shared::error::info_err!("Failed to open Xtream tree update: {e}"))?;
|
||||
|
||||
if let Some(mut item) = tree_update
|
||||
.query(&provider_id)
|
||||
.map_err(|e| shared::error::info_err!("Tree query error: {e}"))?
|
||||
{
|
||||
update_properties(
|
||||
&mut item.additional_properties,
|
||||
item_type,
|
||||
&item.name,
|
||||
item.virtual_id,
|
||||
raw_video,
|
||||
raw_audio,
|
||||
);
|
||||
tree_update
|
||||
.update(&provider_id, item)
|
||||
.map_err(|e| shared::error::info_err!("Tree update error: {e}"))?;
|
||||
info!("Successfully updated Xtream metadata for: {unique_id}");
|
||||
} else {
|
||||
warn!("Item not found in Xtream DB: {unique_id}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn update_properties(
|
||||
props_opt: &mut Option<StreamProperties>,
|
||||
item_type: PlaylistItemType,
|
||||
name: &str,
|
||||
virtual_id: u32,
|
||||
raw_video: Option<serde_json::Value>,
|
||||
raw_audio: Option<serde_json::Value>
|
||||
) {
|
||||
if matches!(item_type, PlaylistItemType::Video | PlaylistItemType::LocalVideo) {
|
||||
let mut props = if let Some(StreamProperties::Video(p)) = props_opt {
|
||||
*p.clone()
|
||||
} else {
|
||||
VideoStreamProperties {
|
||||
name: name.into(),
|
||||
stream_id: virtual_id,
|
||||
container_extension: "".into(),
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
|
||||
if props.details.is_none() {
|
||||
props.details = Some(VideoStreamDetailProperties::default());
|
||||
}
|
||||
if let Some(details) = props.details.as_mut() {
|
||||
if let Some(v) = raw_video {
|
||||
details.video = Some(v.to_string().into());
|
||||
}
|
||||
if let Some(a) = raw_audio {
|
||||
details.audio = Some(a.to_string().into());
|
||||
}
|
||||
}
|
||||
*props_opt = Some(StreamProperties::Video(Box::new(props)));
|
||||
}
|
||||
else if matches!(item_type, PlaylistItemType::Series | PlaylistItemType::LocalSeries) {
|
||||
let mut props = if let Some(StreamProperties::Episode(p)) = props_opt {
|
||||
*p.clone()
|
||||
} else {
|
||||
EpisodeStreamProperties {
|
||||
episode_id: virtual_id,
|
||||
episode: 0,
|
||||
season: 0,
|
||||
added: None,
|
||||
release_date: None,
|
||||
series_release_date: None,
|
||||
tmdb: None,
|
||||
movie_image: "".into(),
|
||||
container_extension: "".into(),
|
||||
video: None,
|
||||
audio: None,
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(v) = raw_video {
|
||||
props.video = Some(v.to_string().into());
|
||||
}
|
||||
if let Some(a) = raw_audio {
|
||||
props.audio = Some(a.to_string().into());
|
||||
}
|
||||
*props_opt = Some(StreamProperties::Episode(Box::new(props)));
|
||||
}
|
||||
else if matches!(item_type, PlaylistItemType::Live | PlaylistItemType::LiveHls | PlaylistItemType::LiveDash) {
|
||||
let mut props = if let Some(StreamProperties::Live(p)) = props_opt {
|
||||
*p.clone()
|
||||
} else {
|
||||
LiveStreamProperties {
|
||||
name: name.into(),
|
||||
stream_id: virtual_id,
|
||||
..LiveStreamProperties::default()
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(v) = raw_video {
|
||||
props.video = Some(v.to_string().into());
|
||||
}
|
||||
if let Some(a) = raw_audio {
|
||||
props.audio = Some(a.to_string().into());
|
||||
}
|
||||
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
props.last_probed_timestamp = Some(now);
|
||||
props.last_success_timestamp = Some(now);
|
||||
|
||||
*props_opt = Some(StreamProperties::Live(Box::new(props)));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn library_probe_does_not_require_provider_connection() {
|
||||
assert!(!requires_provider_connection_for_generic_probe(InputType::Library));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn m3u_probe_requires_provider_connection() {
|
||||
assert!(requires_provider_connection_for_generic_probe(InputType::M3u));
|
||||
assert!(requires_provider_connection_for_generic_probe(
|
||||
InputType::M3uBatch
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xtream_probe_requires_provider_connection() {
|
||||
assert!(requires_provider_connection_for_generic_probe(
|
||||
InputType::Xtream
|
||||
));
|
||||
assert!(requires_provider_connection_for_generic_probe(
|
||||
InputType::XtreamBatch
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -1,33 +1,138 @@
|
||||
use std::sync::Arc;
|
||||
use shared::error::{info_err};
|
||||
use shared::error::{TuliproxError};
|
||||
use crate::model::{AppConfig, ConfigInput, InputSource};
|
||||
use shared::model::{PlaylistEntry, PlaylistItem, XtreamCluster};
|
||||
use crate::utils::xtream;
|
||||
use crate::model::{AppConfig, ConfigInput};
|
||||
use shared::model::{LiveStreamProperties, StreamProperties, XtreamCluster, XtreamPlaylistItem};
|
||||
use crate::repository::{get_input_storage_path, persist_input_live_info, BPlusTreeQuery, xtream_get_file_path};
|
||||
use crate::utils::{debug_if_enabled};
|
||||
use log::{debug, warn};
|
||||
use crate::processing::parser::xtream::create_xtream_url;
|
||||
use crate::api::model::ProviderIdType;
|
||||
|
||||
pub(in crate::processing) async fn playlist_resolve_download_playlist_item(
|
||||
/// Updates metadata for a single Live stream (primarily probing)
|
||||
pub async fn update_live_stream_metadata(
|
||||
app_config: &Arc<AppConfig>,
|
||||
client: &reqwest::Client,
|
||||
pli: &PlaylistItem,
|
||||
input: &ConfigInput,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
resolve_delay: u16,
|
||||
cluster: XtreamCluster,
|
||||
) -> Option<String> {
|
||||
let mut result = None;
|
||||
let provider_id = pli.get_provider_id()?;
|
||||
if let Some(info_url) = xtream::get_xtream_player_api_info_url(input, cluster, provider_id) {
|
||||
let input_source = InputSource::from(input).with_url(info_url);
|
||||
result = match xtream::get_xtream_stream_info_content(app_config, client, &input_source, true).await {
|
||||
Ok(content) => Some(content),
|
||||
Err(err) => {
|
||||
errors.push(info_err!("{err}"));
|
||||
None
|
||||
id: ProviderIdType,
|
||||
save: bool,
|
||||
db_query: Option<&mut BPlusTreeQuery<u32, XtreamPlaylistItem>>,
|
||||
) -> Result<Option<LiveStreamProperties>, TuliproxError> {
|
||||
let working_dir = &app_config.config.load().working_dir;
|
||||
let storage_path = get_input_storage_path(&input.name, working_dir).await
|
||||
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
|
||||
|
||||
// Try to load existing info first to preserve data
|
||||
let mut props: Option<LiveStreamProperties> = None;
|
||||
let mut existing_item: Option<XtreamPlaylistItem> = None;
|
||||
|
||||
let stream_id_opt = if let ProviderIdType::Id(vid) = id { Some(vid) } else { None };
|
||||
|
||||
if let Some(stream_id) = stream_id_opt {
|
||||
// Use provided query or open new one
|
||||
if let Some(query) = db_query {
|
||||
if let Ok(Some(item)) = query.query_zero_copy(&stream_id) {
|
||||
existing_item = Some(item.clone());
|
||||
if let Some(StreamProperties::Live(p)) = item.additional_properties.as_ref() {
|
||||
props = Some(*p.clone());
|
||||
}
|
||||
}
|
||||
};
|
||||
} else {
|
||||
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Live);
|
||||
if xtream_path.exists() {
|
||||
let _file_lock = app_config.file_locks.read_lock(&xtream_path).await;
|
||||
if let Ok(mut query) = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path) {
|
||||
if let Ok(Some(item)) = query.query_zero_copy(&stream_id) {
|
||||
existing_item = Some(item.clone());
|
||||
if let Some(StreamProperties::Live(p)) = item.additional_properties.as_ref() {
|
||||
props = Some(*p.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if resolve_delay > 0 {
|
||||
tokio::time::sleep(std::time::Duration::new(u64::from(resolve_delay), 0)).await;
|
||||
|
||||
// Initialize props if missing
|
||||
let mut properties = if let Some(p) = props {
|
||||
p
|
||||
} else {
|
||||
LiveStreamProperties {
|
||||
stream_id: stream_id_opt.unwrap_or(0),
|
||||
// If item exists but no props, try to recover name
|
||||
name: existing_item.as_ref().map_or_else(|| "".into(), |i| i.name.clone()),
|
||||
..LiveStreamProperties::default()
|
||||
}
|
||||
};
|
||||
|
||||
// 1. Create dummy properties to generate URL for probing
|
||||
// We construct a temporary property object just for URL generation to ensure we match the stream config
|
||||
// (prefix/extension) correctly, even if we modify `properties` later.
|
||||
let temp_stream_prop = StreamProperties::Live(Box::new(properties.clone()));
|
||||
|
||||
let input_url = input.url.as_str();
|
||||
let username = input.username.as_deref().unwrap_or("");
|
||||
let password = input.password.as_deref().unwrap_or("");
|
||||
let opts = input.options.as_ref();
|
||||
let use_prefix = opts.is_none_or(|o| o.xtream_live_stream_use_prefix);
|
||||
let no_ext = opts.is_some_and(|o| o.xtream_live_stream_without_extension);
|
||||
|
||||
// We generate the URL to probe directly on the provider
|
||||
let stream_url = create_xtream_url(
|
||||
XtreamCluster::Live,
|
||||
input_url, username, password,
|
||||
&temp_stream_prop,
|
||||
use_prefix, no_ext
|
||||
);
|
||||
let config = app_config.config.load();
|
||||
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
|
||||
let user_agent = config.default_user_agent.clone();
|
||||
let analyze_duration = 5_000_000;
|
||||
let probe_size = 5_000_000;
|
||||
|
||||
let display_id = stream_id_opt.map_or_else(|| "StringID".to_string(), |v| v.to_string());
|
||||
debug!("Probing Live Stream ID {} for input {}", display_id, input.name);
|
||||
|
||||
// Update last_probed_timestamp BEFORE probing to ensure we record the attempt even if it crashes/panics (unlikely but safe)
|
||||
// Actually, update it before persisting.
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
properties.last_probed_timestamp = Some(now);
|
||||
|
||||
let mut success = false;
|
||||
if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url(
|
||||
&stream_url,
|
||||
user_agent.as_deref(),
|
||||
analyze_duration,
|
||||
probe_size,
|
||||
ffprobe_timeout,
|
||||
).await {
|
||||
// 3. Update properties on success
|
||||
if let Some(v) = raw_video {
|
||||
properties.video = Some(v.to_string().into());
|
||||
}
|
||||
if let Some(a) = raw_audio {
|
||||
properties.audio = Some(a.to_string().into());
|
||||
}
|
||||
properties.last_success_timestamp = Some(now);
|
||||
success = true;
|
||||
|
||||
debug_if_enabled!("Successfully probed Live Stream ID {}", display_id);
|
||||
} else {
|
||||
warn!("Probe failed for Live Stream ID {} (Input: {})", display_id, input.name);
|
||||
// We still persist the updated last_probed_timestamp so we don't retry immediately
|
||||
}
|
||||
result
|
||||
|
||||
// 4. Persist
|
||||
if save {
|
||||
if let Some(stream_id) = stream_id_opt {
|
||||
persist_input_live_info(app_config, &storage_path, XtreamCluster::Live, &input.name, stream_id, &properties)
|
||||
.await
|
||||
.map_err(|e| shared::error::info_err!("Persist error: {e}"))?;
|
||||
}
|
||||
}
|
||||
|
||||
if !success {
|
||||
// Return error to propagate failure up to task manager/logs
|
||||
return Err(shared::error::info_err!("Probe failed for stream {display_id}"));
|
||||
}
|
||||
|
||||
Ok(Some(properties))
|
||||
}
|
||||
|
||||
@@ -1,260 +1,113 @@
|
||||
use crate::api::model::UpdateTask;
|
||||
use crate::api::model::{ActiveProviderManager, ProviderHandle, ProviderIdType, ResolveReason, ResolveReasonSet};
|
||||
use crate::library::MetadataResolver;
|
||||
use crate::model::FetchedPlaylist;
|
||||
use crate::model::{AppConfig, ConfigTarget};
|
||||
use crate::model::{ConfigInput, InputSource};
|
||||
use crate::processing::parser::xtream::create_xtream_series_episode_url;
|
||||
use crate::processing::parser::xtream::parse_xtream_series_info;
|
||||
use crate::processing::processor::create_resolve_options_function_for_xtream_target;
|
||||
use crate::processing::processor::playlist::ProcessingPipe;
|
||||
use crate::processing::processor::xtream::playlist_resolve_download_playlist_item;
|
||||
use crate::repository::{get_input_storage_path, persist_input_series_info_batch, MemoryPlaylistSource, PlaylistSource};
|
||||
use log::{error, info, log_enabled, Level};
|
||||
use crate::processing::processor::playlist::{PlaylistProcessingContext, ProcessingPipe};
|
||||
use crate::processing::processor::{create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags};
|
||||
use crate::ptt::ptt_parse_title;
|
||||
use crate::repository::persists_input_series_info;
|
||||
use crate::repository::{
|
||||
get_input_storage_path, persist_input_series_info_batch, MemoryPlaylistSource, PlaylistSource,
|
||||
};
|
||||
use crate::repository::{xtream_get_file_path, BPlusTreeQuery};
|
||||
use crate::utils::{debug_if_enabled, xtream};
|
||||
use log::{debug, error, info, log_enabled, trace, warn, Level};
|
||||
use serde_json::Value;
|
||||
use shared::error::TuliproxError;
|
||||
use shared::model::{InputType, PlaylistEntry, SeriesStreamProperties, StreamProperties, XtreamSeriesInfo};
|
||||
use shared::model::{PlaylistGroup, PlaylistItemType, UUIDType, XtreamCluster};
|
||||
use indexmap::IndexMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use shared::model::{MediaQuality, PlaylistEntry, PlaylistItem, SeriesStreamProperties, StreamProperties, XtreamPlaylistItem, XtreamSeriesInfo};
|
||||
use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
create_resolve_options_function_for_xtream_target!(series);
|
||||
|
||||
const BATCH_SIZE: usize = 100;
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn playlist_resolve_series_info(app_config: &Arc<AppConfig>, client: &reqwest::Client,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
fpl: &mut FetchedPlaylist<'_>,
|
||||
resolve_series: bool,
|
||||
resolve_delay: u16) -> Vec<PlaylistGroup> {
|
||||
|
||||
let input = fpl.input;
|
||||
let working_dir = &app_config.config.load().working_dir;
|
||||
let storage_path = match get_input_storage_path(&input.name, working_dir).await {
|
||||
Ok(storage_path) => storage_path,
|
||||
Err(err) => {
|
||||
error!("Can't resolve series info, input storage directory for input '{}' failed: {err}", input.name);
|
||||
return vec![];
|
||||
}
|
||||
};
|
||||
|
||||
let series_info_count = if resolve_series {
|
||||
let series_info_count = fpl.get_missing_series_info_count();
|
||||
if series_info_count > 0 {
|
||||
info!("Found {series_info_count} series info to resolve");
|
||||
}
|
||||
series_info_count
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let mut last_log_time = Instant::now();
|
||||
let mut processed_series_info_count = 0;
|
||||
let mut group_series: IndexMap<u32, PlaylistGroup> = IndexMap::new();
|
||||
let mut batch = Vec::with_capacity(BATCH_SIZE);
|
||||
|
||||
let input = fpl.input;
|
||||
for pli in fpl.items_mut() {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Series
|
||||
|| pli.header.item_type != PlaylistItemType::SeriesInfo {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(provider_id) = pli.get_provider_id() else { continue; };
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let should_download = resolve_series && !pli.has_details();
|
||||
if should_download {
|
||||
processed_series_info_count += 1;
|
||||
if let Some(content) = playlist_resolve_download_playlist_item(
|
||||
app_config,
|
||||
client,
|
||||
pli,
|
||||
input,
|
||||
errors,
|
||||
resolve_delay,
|
||||
XtreamCluster::Series,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if !content.is_empty() {
|
||||
//tokio::fs::write(&storage_path.join(format!("{provider_id}_series_info.json")), &content).await.ok();
|
||||
match serde_json::from_str::<XtreamSeriesInfo>(&content) {
|
||||
Ok(info) => {
|
||||
let series_stream_props = SeriesStreamProperties::from_info(&info, pli);
|
||||
|
||||
batch.push((provider_id, series_stream_props.clone()));
|
||||
if batch.len() >= BATCH_SIZE {
|
||||
if let Err(err) = persist_input_series_info_batch(app_config, &storage_path, XtreamCluster::Series, &input.name, std::mem::take(&mut batch)).await {
|
||||
error!("Failed to persist batch series info: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
// Update in-memory playlist items with the newly fetched vod info.
|
||||
// This makes the data available for later processing steps like STRM export.
|
||||
pli.header.additional_properties = Some(StreamProperties::Series(Box::new(series_stream_props)));
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to parse series info for provider_id {provider_id}: {err}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extract episodes from info
|
||||
if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() {
|
||||
let (group, series_name) = {
|
||||
let header = &pli.header;
|
||||
(header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() })
|
||||
};
|
||||
if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input) {
|
||||
let group = group_series.entry(pli.header.category_id)
|
||||
.or_insert_with(|| {
|
||||
PlaylistGroup {
|
||||
id: pli.header.category_id,
|
||||
title: pli.header.group.clone(),
|
||||
channels: Vec::new(),
|
||||
xtream_cluster: XtreamCluster::Series,
|
||||
}
|
||||
});
|
||||
group.channels.extend(episodes.into_iter());
|
||||
}
|
||||
}
|
||||
|
||||
if resolve_series && log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 {
|
||||
info!("resolved {processed_series_info_count}/{series_info_count} series info");
|
||||
last_log_time = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
if !batch.is_empty() {
|
||||
if let Err(err) = persist_input_series_info_batch(app_config, &storage_path, XtreamCluster::Series, &input.name, batch).await {
|
||||
error!("Failed to persist final batch series info: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
if resolve_series {
|
||||
info!("resolved {processed_series_info_count}/{series_info_count} series info");
|
||||
}
|
||||
group_series.into_values().collect()
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub async fn playlist_resolve_series(cfg: &Arc<AppConfig>,
|
||||
client: &reqwest::Client,
|
||||
target: &ConfigTarget,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
pipe: &ProcessingPipe,
|
||||
provider_fpl: &mut FetchedPlaylist<'_>,
|
||||
processed_fpl: &mut FetchedPlaylist<'_>,
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn playlist_resolve_series(
|
||||
ctx: &PlaylistProcessingContext,
|
||||
target: &ConfigTarget,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
pipe: &ProcessingPipe,
|
||||
provider_fpl: &mut FetchedPlaylist<'_>,
|
||||
processed_fpl: &mut FetchedPlaylist<'_>,
|
||||
) {
|
||||
let (resolve_series, resolve_delay) = get_resolve_series_options(target, processed_fpl);
|
||||
let resolve_options = get_resolve_series_options(target, processed_fpl);
|
||||
|
||||
let do_probe = resolve_options.flags.contains(ResolveOptionsFlags::Probe) && ctx.config.is_ffprobe_enabled().await;
|
||||
|
||||
provider_fpl.source.release_resources(XtreamCluster::Series);
|
||||
let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_series, resolve_delay).await;
|
||||
provider_fpl.source.obtain_resources().await;
|
||||
|
||||
// Keep input playlist as canonical source:
|
||||
// - Store resolved series details back into provider_fpl items
|
||||
// - Rebuild base (unmodified) episodes from provider_fpl metadata only
|
||||
playlist_resolve_series_info(
|
||||
ctx,
|
||||
errors,
|
||||
processed_fpl,
|
||||
resolve_options,
|
||||
do_probe,
|
||||
pipe,
|
||||
target,
|
||||
)
|
||||
.await;
|
||||
|
||||
if provider_fpl.is_memory() {
|
||||
let mut resolved_series_by_provider_id: HashMap<u32, SeriesStreamProperties> = HashMap::new();
|
||||
for pli in processed_fpl.items() {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Series
|
||||
|| pli.header.item_type != PlaylistItemType::SeriesInfo
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(provider_id) = pli.get_provider_id() else { continue; };
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() {
|
||||
resolved_series_by_provider_id
|
||||
.entry(provider_id)
|
||||
.or_insert_with(|| properties.as_ref().clone());
|
||||
}
|
||||
}
|
||||
|
||||
if !resolved_series_by_provider_id.is_empty() {
|
||||
let mut existing_uuids: HashSet<UUIDType> = provider_fpl
|
||||
.items()
|
||||
.filter(|pli| {
|
||||
pli.header.xtream_cluster == XtreamCluster::Series
|
||||
&& pli.header.item_type == PlaylistItemType::Series
|
||||
})
|
||||
.map(|pli| pli.get_uuid())
|
||||
.collect();
|
||||
|
||||
let mut base_series_groups: IndexMap<u32, PlaylistGroup> = IndexMap::new();
|
||||
let input = provider_fpl.input;
|
||||
for pli in provider_fpl.items_mut() {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Series
|
||||
|| pli.header.item_type != PlaylistItemType::SeriesInfo
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(provider_id) = pli.get_provider_id() else { continue; };
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(resolved) = resolved_series_by_provider_id.get(&provider_id) {
|
||||
pli.header.additional_properties = Some(StreamProperties::Series(Box::new(resolved.clone())));
|
||||
}
|
||||
|
||||
let Some(StreamProperties::Series(series_props)) = pli.header.additional_properties.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let (category_id, group_title, series_name) = {
|
||||
let header = &pli.header;
|
||||
(
|
||||
header.category_id,
|
||||
header.group.clone(),
|
||||
if header.name.is_empty() {
|
||||
header.title.clone()
|
||||
} else {
|
||||
header.name.clone()
|
||||
},
|
||||
)
|
||||
};
|
||||
|
||||
if let Some(episodes) = parse_xtream_series_info(
|
||||
&pli.get_uuid(),
|
||||
series_props,
|
||||
&group_title,
|
||||
&series_name,
|
||||
input,
|
||||
) {
|
||||
let group = base_series_groups.entry(category_id).or_insert_with(|| PlaylistGroup {
|
||||
id: category_id,
|
||||
title: group_title,
|
||||
channels: Vec::new(),
|
||||
xtream_cluster: XtreamCluster::Series,
|
||||
});
|
||||
for episode in episodes {
|
||||
if existing_uuids.insert(episode.get_uuid()) {
|
||||
group.channels.push(episode);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for group in base_series_groups.values().filter(|g| !g.channels.is_empty()) {
|
||||
provider_fpl.update_playlist(group).await;
|
||||
}
|
||||
}
|
||||
sync_resolved_series_properties(provider_fpl, processed_fpl);
|
||||
}
|
||||
|
||||
if series_playlist.is_empty() {
|
||||
return;
|
||||
}
|
||||
provider_fpl.source.obtain_resources().await;
|
||||
}
|
||||
|
||||
// run the target processing pipe over newly derived episodes
|
||||
// run the processing pipe over new items
|
||||
let mut new_playlist = series_playlist;
|
||||
#[allow(clippy::too_many_lines, clippy::too_many_arguments)]
|
||||
async fn playlist_resolve_series_info(ctx: &PlaylistProcessingContext,
|
||||
_errors: &mut Vec<TuliproxError>,
|
||||
fpl: &mut FetchedPlaylist<'_>,
|
||||
resolve_options: ResolveOptions,
|
||||
do_probe: bool,
|
||||
pipe: &ProcessingPipe,
|
||||
target: &ConfigTarget) {
|
||||
let filter = |pli: &PlaylistItem| {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Series
|
||||
|| pli.header.item_type != PlaylistItemType::SeriesInfo
|
||||
{
|
||||
return false;
|
||||
}
|
||||
true
|
||||
};
|
||||
|
||||
// Skip if nothing to do
|
||||
let skip_resolve = !resolve_options.flags.contains(ResolveOptionsFlags::Resolve) && !do_probe && !resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing);
|
||||
|
||||
let resolve_tmdb_enabled = fpl.input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
|
||||
|
||||
let groups_to_add = if resolve_options.flags.contains(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() {
|
||||
queue_background_series_info(
|
||||
ctx,
|
||||
fpl,
|
||||
filter,
|
||||
&resolve_options,
|
||||
do_probe,
|
||||
skip_resolve,
|
||||
resolve_tmdb_enabled,
|
||||
)
|
||||
} else {
|
||||
process_immediate_series_info(
|
||||
ctx,
|
||||
fpl,
|
||||
filter,
|
||||
&resolve_options,
|
||||
do_probe,
|
||||
skip_resolve,
|
||||
resolve_tmdb_enabled,
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
||||
// Apply pipe transformations to new groups
|
||||
let mut new_playlist = groups_to_add;
|
||||
for f in pipe {
|
||||
let mut source = MemoryPlaylistSource::new(new_playlist);
|
||||
if let Some(v) = f(&mut source, target) {
|
||||
@@ -264,8 +117,628 @@ pub async fn playlist_resolve_series(cfg: &Arc<AppConfig>,
|
||||
}
|
||||
}
|
||||
|
||||
// assign new items to the new playlist
|
||||
for plg in &new_playlist {
|
||||
processed_fpl.update_playlist(plg).await;
|
||||
// Apply resolved episodes to playlist
|
||||
for group in new_playlist {
|
||||
fpl.update_playlist(&group).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn sync_resolved_series_properties(
|
||||
provider_fpl: &mut FetchedPlaylist<'_>,
|
||||
processed_fpl: &mut FetchedPlaylist<'_>,
|
||||
) {
|
||||
let mut resolved_series_by_provider_id: HashMap<u32, SeriesStreamProperties> = HashMap::new();
|
||||
|
||||
for pli in processed_fpl.items() {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Series
|
||||
|| pli.header.item_type != PlaylistItemType::SeriesInfo
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(provider_id) = pli.get_provider_id() else {
|
||||
continue;
|
||||
};
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() {
|
||||
resolved_series_by_provider_id
|
||||
.entry(provider_id)
|
||||
.or_insert_with(|| properties.as_ref().clone());
|
||||
}
|
||||
}
|
||||
|
||||
if resolved_series_by_provider_id.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
for source_pli in provider_fpl.items_mut() {
|
||||
if source_pli.header.xtream_cluster != XtreamCluster::Series
|
||||
|| source_pli.header.item_type != PlaylistItemType::SeriesInfo
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(provider_id) = source_pli.get_provider_id() else {
|
||||
continue;
|
||||
};
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(resolved) = resolved_series_by_provider_id.get(&provider_id) {
|
||||
source_pli.header.additional_properties =
|
||||
Some(StreamProperties::Series(Box::new(resolved.clone())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn queue_background_series_info(
|
||||
ctx: &PlaylistProcessingContext,
|
||||
fpl: &mut FetchedPlaylist<'_>,
|
||||
filter: impl Fn(&PlaylistItem) -> bool,
|
||||
resolve_options: &ResolveOptions,
|
||||
do_probe: bool,
|
||||
skip_resolve: bool,
|
||||
resolve_tmdb_enabled: bool,
|
||||
) -> Vec<PlaylistGroup> {
|
||||
let mut groups_to_add = Vec::new();
|
||||
let Some(mgr) = ctx.metadata_manager.as_ref() else {
|
||||
return groups_to_add;
|
||||
};
|
||||
|
||||
let input = fpl.input;
|
||||
let input_name_arc = input.name.clone();
|
||||
|
||||
for pli in fpl.items_mut() {
|
||||
if !filter(pli) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let provider_id = if let Ok(uid) = pli.header.id.parse::<u32>() {
|
||||
ProviderIdType::Id(uid)
|
||||
} else {
|
||||
ProviderIdType::from(pli.header.id.to_string())
|
||||
};
|
||||
|
||||
if !skip_resolve {
|
||||
let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli);
|
||||
|
||||
if !reasons.is_empty() {
|
||||
let task = UpdateTask::ResolveSeries {
|
||||
id: provider_id,
|
||||
reason: reasons,
|
||||
delay: resolve_options.resolve_delay,
|
||||
};
|
||||
mgr.queue_task_background(input_name_arc.clone(), task);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(group_obj) = expand_series_item(pli, input) {
|
||||
groups_to_add.push(group_obj);
|
||||
}
|
||||
}
|
||||
groups_to_add
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn process_immediate_series_info(
|
||||
ctx: &PlaylistProcessingContext,
|
||||
fpl: &mut FetchedPlaylist<'_>,
|
||||
filter: impl Fn(&PlaylistItem) -> bool,
|
||||
resolve_options: &ResolveOptions,
|
||||
do_probe: bool,
|
||||
skip_resolve: bool,
|
||||
resolve_tmdb_enabled: bool,
|
||||
) -> Vec<PlaylistGroup> {
|
||||
let input = fpl.input;
|
||||
let working_dir = &ctx.config.config.load().working_dir;
|
||||
let storage_path = match get_input_storage_path(&input.name, working_dir).await {
|
||||
Ok(path) => path,
|
||||
Err(err) => {
|
||||
error!("Can't resolve series, input storage directory for input '{}' failed: {err}", input.name);
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
// Keep an optional read query open and reopen lazily only when needed.
|
||||
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Series);
|
||||
let mut db_query_holder = None;
|
||||
let mut _db_lock_holder = None;
|
||||
|
||||
let mut groups_to_add = Vec::new();
|
||||
let mut batch: Vec<(ProviderIdType, SeriesStreamProperties)> = Vec::with_capacity(BATCH_SIZE);
|
||||
let mut processed_count = 0;
|
||||
let mut last_log_time = Instant::now();
|
||||
|
||||
for pli in fpl.items_mut() {
|
||||
if !filter(pli) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let provider_id = if let Ok(uid) = pli.header.id.parse::<u32>() {
|
||||
ProviderIdType::Id(uid)
|
||||
} else {
|
||||
ProviderIdType::from(&*pli.header.id)
|
||||
};
|
||||
|
||||
if !skip_resolve {
|
||||
let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli);
|
||||
|
||||
if !reasons.is_empty() {
|
||||
if let Some(active_provider) = ctx.provider_manager.as_ref() {
|
||||
if db_query_holder.is_none() && xtream_path.exists() {
|
||||
let lock = ctx.config.file_locks.read_lock(&xtream_path).await;
|
||||
if let Ok(query) = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path) {
|
||||
db_query_holder = Some(query);
|
||||
_db_lock_holder = Some(lock);
|
||||
}
|
||||
}
|
||||
// Pass the optional reference to the query
|
||||
let db_query_ref = db_query_holder.as_mut();
|
||||
|
||||
match update_series_info_immediate(ctx, active_provider, input, pli, provider_id.clone(), &reasons, db_query_ref).await {
|
||||
Ok(Some(updated_props)) => {
|
||||
pli.header.additional_properties = Some(StreamProperties::Series(Box::new(updated_props.clone())));
|
||||
batch.push((provider_id.clone(), updated_props));
|
||||
|
||||
if batch.len() >= BATCH_SIZE {
|
||||
// Release lock before persisting to avoid deadlock (persist needs write lock)
|
||||
db_query_holder = None;
|
||||
_db_lock_holder = None;
|
||||
|
||||
let updates: Vec<(u32, SeriesStreamProperties)> = batch.iter()
|
||||
.filter_map(|(id, props)| {
|
||||
if let ProviderIdType::Id(vid) = id {
|
||||
Some((*vid, props.clone()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if updates.is_empty() {
|
||||
batch.clear();
|
||||
} else {
|
||||
match persist_input_series_info_batch(
|
||||
&ctx.config,
|
||||
&storage_path,
|
||||
XtreamCluster::Series,
|
||||
&input.name,
|
||||
updates,
|
||||
).await {
|
||||
Ok(()) => batch.clear(),
|
||||
Err(err) => {
|
||||
error!(
|
||||
"persist_input_series_info_batch failed for XtreamCluster::Series on input '{}'. batch.clear() skipped. Error: {}",
|
||||
input.name, err
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
processed_count += 1;
|
||||
if resolve_options.resolve_delay > 0 {
|
||||
tokio::time::sleep(Duration::from_secs(u64::from(resolve_options.resolve_delay))).await;
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => {
|
||||
error!("Failed to update Series metadata for {}: {e}", pli.header.title);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(group_obj) = expand_series_item(pli, input) {
|
||||
groups_to_add.push(group_obj);
|
||||
}
|
||||
|
||||
if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 {
|
||||
info!("resolved {processed_count} series info");
|
||||
last_log_time = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
if !batch.is_empty() {
|
||||
// Release lock before final persist
|
||||
_db_lock_holder = None;
|
||||
|
||||
let updates: Vec<(u32, SeriesStreamProperties)> = batch.into_iter()
|
||||
.filter_map(|(id, props)| if let ProviderIdType::Id(vid) = id { Some((vid, props)) } else { None })
|
||||
.collect();
|
||||
|
||||
if !updates.is_empty() {
|
||||
if let Err(err) = persist_input_series_info_batch(
|
||||
&ctx.config,
|
||||
&storage_path,
|
||||
XtreamCluster::Series,
|
||||
&input.name,
|
||||
updates,
|
||||
).await {
|
||||
error!("Failed to persist final batch series info: {err}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if processed_count > 0 {
|
||||
info!("Processed {processed_count} series info");
|
||||
}
|
||||
|
||||
groups_to_add
|
||||
}
|
||||
|
||||
fn expand_series_item(pli: &PlaylistItem, input: &ConfigInput) -> Option<PlaylistGroup> {
|
||||
if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() {
|
||||
let global_release_date = properties.release_date.clone();
|
||||
let (group, series_name) = {
|
||||
let header = &pli.header;
|
||||
(header.group.clone(), header.get_name())
|
||||
};
|
||||
|
||||
if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input, global_release_date.as_ref()) {
|
||||
return Some(PlaylistGroup {
|
||||
id: pli.header.category_id,
|
||||
title: pli.header.group.clone(),
|
||||
channels: episodes,
|
||||
xtream_cluster: XtreamCluster::Series,
|
||||
});
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn update_series_info_immediate(
|
||||
ctx: &PlaylistProcessingContext,
|
||||
active_provider: &Arc<ActiveProviderManager>,
|
||||
input: &ConfigInput,
|
||||
pli: &PlaylistItem,
|
||||
id: ProviderIdType,
|
||||
reasons: &ResolveReasonSet,
|
||||
db_query: Option<&mut BPlusTreeQuery<u32, XtreamPlaylistItem>>,
|
||||
) -> Result<Option<SeriesStreamProperties>, TuliproxError> {
|
||||
let fetch_info = reasons.contains(ResolveReason::Info);
|
||||
|
||||
update_series_metadata(
|
||||
&ctx.config,
|
||||
&ctx.client,
|
||||
input,
|
||||
id,
|
||||
active_provider,
|
||||
None, // active_handle
|
||||
Some(&pli.header.title),
|
||||
false, // save (we batch in caller)
|
||||
fetch_info,
|
||||
reasons.contains(ResolveReason::Probe),
|
||||
db_query,
|
||||
).await
|
||||
}
|
||||
|
||||
fn check_resolve_reasons(
|
||||
resolve_options: &ResolveOptions,
|
||||
do_probe: bool,
|
||||
resolve_tmdb_enabled: bool,
|
||||
pli: &mut PlaylistItem,
|
||||
) -> ResolveReasonSet {
|
||||
let mut reasons = ResolveReasonSet::new();
|
||||
|
||||
let needs_info = check_needs_info(resolve_options, pli, &mut reasons);
|
||||
check_resolve_tmdb(resolve_options, resolve_tmdb_enabled, pli, needs_info, &mut reasons);
|
||||
|
||||
if do_probe {
|
||||
check_needs_probe(pli, &mut reasons);
|
||||
}
|
||||
reasons
|
||||
}
|
||||
|
||||
fn check_needs_info(resolve_options: &ResolveOptions, pli: &mut PlaylistItem, reasons: &mut ResolveReasonSet) -> bool {
|
||||
let needs_info = resolve_options.flags.contains(ResolveOptionsFlags::Resolve) && !pli.has_details();
|
||||
if needs_info {
|
||||
reasons.add(ResolveReason::Info);
|
||||
}
|
||||
needs_info
|
||||
}
|
||||
|
||||
fn check_resolve_tmdb(
|
||||
resolve_options: &ResolveOptions,
|
||||
resolve_tmdb_enabled: bool,
|
||||
pli: &mut PlaylistItem,
|
||||
needs_info: bool,
|
||||
reasons: &mut ResolveReasonSet,
|
||||
) {
|
||||
if resolve_tmdb_enabled && resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing) {
|
||||
if let Some(StreamProperties::Series(series_stream_props)) = pli.header.additional_properties.as_ref() {
|
||||
let has_tmdb = series_stream_props.tmdb.is_some();
|
||||
let has_date = series_stream_props.release_date.is_some();
|
||||
let title_present = !series_stream_props.name.is_empty() || !pli.header.title.is_empty();
|
||||
|
||||
if title_present && (!has_tmdb || !has_date) {
|
||||
if !has_tmdb { reasons.add(ResolveReason::Tmdb); }
|
||||
if !has_date { reasons.add(ResolveReason::Date); }
|
||||
}
|
||||
} else if needs_info {
|
||||
// Covered by info
|
||||
} else {
|
||||
reasons.add(ResolveReason::MissingDetails);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_needs_probe(pli: &mut PlaylistItem, reasons: &mut ResolveReasonSet) {
|
||||
if let Some(StreamProperties::Series(series_stream_props)) = pli.header.additional_properties.as_ref() {
|
||||
if let Some(details) = series_stream_props.details.as_ref() {
|
||||
if let Some(episodes) = details.episodes.as_ref() {
|
||||
for ep in episodes {
|
||||
let missing_video = !MediaQuality::is_valid_media_info(ep.video.as_deref());
|
||||
let missing_audio = !MediaQuality::is_valid_media_info(ep.audio.as_deref());
|
||||
if missing_video || missing_audio {
|
||||
reasons.add(ResolveReason::Probe);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Updates metadata for a single Series (Info + Episodes Probe) and persists it.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `save` - If true, persists changes to the input database immediately (Instant strategy).
|
||||
/// If false, returns the properties so the caller can batch persist them (Bundled strategy).
|
||||
/// * `fetch_info` - If true, fetches details from Provider API. If false, uses existing/dummy data.
|
||||
/// * `db_query` - Optional pre-opened DB handle to avoid re-opening file.
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub async fn update_series_metadata(
|
||||
app_config: &Arc<AppConfig>,
|
||||
client: &reqwest::Client,
|
||||
input: &ConfigInput,
|
||||
id: ProviderIdType,
|
||||
active_provider: &Arc<ActiveProviderManager>,
|
||||
active_handle: Option<&ProviderHandle>,
|
||||
playlist_title: Option<&str>,
|
||||
save: bool,
|
||||
fetch_info: bool,
|
||||
do_probe: bool,
|
||||
db_query: Option<&mut BPlusTreeQuery<u32, XtreamPlaylistItem>>,
|
||||
) -> Result<Option<SeriesStreamProperties>, TuliproxError> {
|
||||
let working_dir = &app_config.config.load().working_dir;
|
||||
let storage_path = get_input_storage_path(&input.name, working_dir).await
|
||||
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
|
||||
|
||||
let opts = input.options.as_ref();
|
||||
if opts.is_some_and(|o| o.xtream_skip_series) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Try to load existing info first
|
||||
let mut props: Option<SeriesStreamProperties> = None;
|
||||
let mut existing_item: Option<XtreamPlaylistItem> = None;
|
||||
|
||||
let series_id_opt = if let ProviderIdType::Id(vid) = id { Some(vid) } else { None };
|
||||
|
||||
if let Some(series_id) = series_id_opt {
|
||||
// Use provided query or open new one
|
||||
if let Some(query) = db_query {
|
||||
if let Ok(Some(item)) = query.query_zero_copy(&series_id) {
|
||||
existing_item = Some(item.clone());
|
||||
if let Some(StreamProperties::Series(p)) = item.additional_properties.as_ref() {
|
||||
props = Some(*p.clone());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Series);
|
||||
if xtream_path.exists() {
|
||||
let _file_lock = app_config.file_locks.read_lock(&xtream_path).await;
|
||||
if let Ok(mut query) = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path) {
|
||||
if let Ok(Some(item)) = query.query_zero_copy(&series_id) {
|
||||
existing_item = Some(item.clone());
|
||||
if let Some(StreamProperties::Series(p)) = item.additional_properties.as_ref() {
|
||||
props = Some(*p.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut fetched_new = false;
|
||||
let mut properties_updated = false;
|
||||
|
||||
let display_id = series_id_opt.map_or_else(|| "StringID".to_string(), |v| v.to_string());
|
||||
|
||||
// Input DB is canonical, but if details are missing we still need one info fetch to complete the record.
|
||||
let should_fetch_info = fetch_info && props.as_ref().is_none_or(|p| p.details.is_none());
|
||||
|
||||
// 1. Fetch Info from Provider (only when missing in input DB)
|
||||
if should_fetch_info {
|
||||
if let Some(series_id) = series_id_opt {
|
||||
// Fetch Info from Provider
|
||||
let info_url = xtream::get_xtream_player_api_info_url(input, XtreamCluster::Series, series_id)
|
||||
.ok_or_else(|| shared::error::info_err!("Failed to build info URL"))?;
|
||||
|
||||
let input_source = InputSource::from(input).with_url(info_url);
|
||||
let content = xtream::get_xtream_stream_info_content(app_config, client, &input_source, true)
|
||||
.await
|
||||
.map_err(|e| shared::error::info_err!("{e}"))?;
|
||||
|
||||
if !content.is_empty() {
|
||||
if let Ok(mut json_value) = serde_json::from_str::<Value>(&content) {
|
||||
if let Some(info) = json_value.get_mut("info").and_then(|v| v.as_object_mut()) {
|
||||
crate::model::normalize_release_date(info);
|
||||
}
|
||||
|
||||
if let Ok(info) = serde_json::from_value::<XtreamSeriesInfo>(json_value) {
|
||||
if let Some(existing) = &existing_item {
|
||||
props = Some(SeriesStreamProperties::from_info(&info, existing));
|
||||
fetched_new = true;
|
||||
properties_updated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we don't have props, verify if we can proceed with minimal dummy props
|
||||
if props.is_none() {
|
||||
if let Some(title) = playlist_title.or_else(|| existing_item.as_ref().map(|i| i.title.as_ref())) {
|
||||
let new_props = SeriesStreamProperties {
|
||||
name: title.into(),
|
||||
series_id: series_id_opt.unwrap_or(0),
|
||||
..Default::default()
|
||||
};
|
||||
props = Some(new_props);
|
||||
} else {
|
||||
return Err(shared::error::info_err!("No Series properties available and no title found for {display_id}"));
|
||||
}
|
||||
}
|
||||
|
||||
let mut properties = props.unwrap();
|
||||
|
||||
let resolve_tmdb_enabled = input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
|
||||
|
||||
// 2. Resolve TMDB/Date if missing
|
||||
if resolve_tmdb_enabled && (properties.tmdb.is_none() || properties.release_date.is_none()) && !properties.name.is_empty() {
|
||||
let config = app_config.config.load();
|
||||
let library_config = config.library.as_ref();
|
||||
let meta_resolver = MetadataResolver::new(library_config, client.clone());
|
||||
|
||||
let mut meta = None;
|
||||
let mut tried_title = false;
|
||||
|
||||
// Try extracting year locally first
|
||||
if properties.release_date.is_none() {
|
||||
let meta_parse = ptt_parse_title(&properties.name);
|
||||
if let Some(year) = meta_parse.year {
|
||||
properties.release_date = Some(format!("{year}-01-01").into());
|
||||
properties_updated = true;
|
||||
debug_if_enabled!("Parsed local year for Series '{}': {}", properties.name, year);
|
||||
}
|
||||
}
|
||||
|
||||
// 1. & 2. Playlist Title
|
||||
let title_candidate = playlist_title.or_else(|| existing_item.as_ref().map(|i| i.title.as_ref()));
|
||||
if let Some(title) = title_candidate {
|
||||
if !title.is_empty() {
|
||||
trace!("Resolving TMDB for Series using Playlist Title '{title}' (ID: {display_id})...");
|
||||
meta = meta_resolver.resolve_from_title(title, properties.tmdb, false, resolve_tmdb_enabled).await;
|
||||
tried_title = true;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. API Name (fallback)
|
||||
if (meta.is_none() || (meta.as_ref().is_some_and(|m| m.tmdb_id().is_none())))
|
||||
&& !properties.name.is_empty() {
|
||||
let title_already_tried = if let Some(t) = title_candidate { t == properties.name.as_ref() } else { false };
|
||||
if !tried_title || !title_already_tried {
|
||||
debug!("Fallback to API Name '{}'...", properties.name);
|
||||
meta = meta_resolver.resolve_from_title(&properties.name, properties.tmdb, false, resolve_tmdb_enabled).await;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(m) = meta {
|
||||
if properties.tmdb.is_none() {
|
||||
properties.tmdb = m.tmdb_id();
|
||||
properties_updated = true;
|
||||
}
|
||||
if properties.release_date.is_none() {
|
||||
properties.release_date = m.year().map(|y| format!("{y}-01-01").into());
|
||||
properties_updated = true;
|
||||
}
|
||||
if properties_updated {
|
||||
let id_display = properties.tmdb.map_or("None".to_string(), |id| id.to_string());
|
||||
debug_if_enabled!("Resolved TMDB for Series ID {}: {}", display_id, id_display);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Probe Episodes (if enabled)
|
||||
if do_probe && app_config.is_ffprobe_enabled().await {
|
||||
if let Some(details) = properties.details.as_mut() {
|
||||
if let Some(episodes) = details.episodes.as_mut() {
|
||||
let config = app_config.config.load();
|
||||
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
|
||||
let user_agent = config.default_user_agent.clone();
|
||||
let analyze_duration = 10_000_000;
|
||||
let probe_size = 10_000_000;
|
||||
|
||||
let input_url = input.url.as_str();
|
||||
let input_username = input.username.as_deref().unwrap_or("");
|
||||
let input_password = input.password.as_deref().unwrap_or("");
|
||||
|
||||
let mut probed_count = 0;
|
||||
|
||||
for ep in episodes {
|
||||
let missing_video = !MediaQuality::is_valid_media_info(ep.video.as_deref());
|
||||
let missing_audio = !MediaQuality::is_valid_media_info(ep.audio.as_deref());
|
||||
|
||||
if missing_video || missing_audio {
|
||||
|
||||
// Acquire Connection logic
|
||||
let temp_handle = if active_handle.is_some() {
|
||||
None
|
||||
} else {
|
||||
active_provider.acquire_connection_for_probe(&input.name).await
|
||||
};
|
||||
|
||||
if active_handle.is_some() || temp_handle.is_some() {
|
||||
let episode_url = create_xtream_series_episode_url(input_url, input_username, input_password, ep);
|
||||
|
||||
// Specific logging for the user to follow
|
||||
let missing_reason = if missing_video && missing_audio { "video/audio" } else if missing_video { "video" } else { "audio" };
|
||||
debug!("Probing Series Episode '{}' (S{}E{}) - Missing {}", ep.title, ep.season, ep.episode_num, missing_reason);
|
||||
|
||||
if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url(
|
||||
&episode_url,
|
||||
user_agent.as_deref(),
|
||||
analyze_duration,
|
||||
probe_size,
|
||||
ffprobe_timeout,
|
||||
).await {
|
||||
if let Some(v) = raw_video {
|
||||
ep.video = Some(v.to_string().into());
|
||||
properties_updated = true;
|
||||
}
|
||||
if let Some(a) = raw_audio {
|
||||
ep.audio = Some(a.to_string().into());
|
||||
properties_updated = true;
|
||||
}
|
||||
probed_count += 1;
|
||||
}
|
||||
|
||||
if let Some(h) = temp_handle {
|
||||
active_provider.release_handle(&h).await;
|
||||
}
|
||||
} else {
|
||||
warn!("Skipping probe for series episode {} due to connection limits", ep.title);
|
||||
}
|
||||
}
|
||||
}
|
||||
if probed_count > 0 {
|
||||
info!("Probed {probed_count} episodes for Series ID {display_id}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Persist
|
||||
if properties_updated || fetched_new {
|
||||
if save {
|
||||
if let Some(series_id) = series_id_opt {
|
||||
persists_input_series_info(app_config, &storage_path, XtreamCluster::Series, &input.name, series_id, &properties)
|
||||
.await
|
||||
.map_err(|e| shared::error::info_err!("Persist error: {e}"))?;
|
||||
|
||||
debug_if_enabled!("Successfully updated Series metadata for ID {}", series_id);
|
||||
}
|
||||
}
|
||||
return Ok(Some(properties));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
@@ -1,129 +1,713 @@
|
||||
use crate::api::model::{ActiveProviderManager, ProviderHandle};
|
||||
use crate::api::model::{ResolveReason, ResolveReasonSet, UpdateTask, ProviderIdType};
|
||||
use crate::library::MetadataResolver;
|
||||
use crate::model::ConfigInput;
|
||||
use crate::model::FetchedPlaylist;
|
||||
use crate::model::InputSource;
|
||||
use crate::model::{AppConfig, ConfigTarget};
|
||||
use crate::processing::processor::create_resolve_options_function_for_xtream_target;
|
||||
use crate::processing::processor::xtream::playlist_resolve_download_playlist_item;
|
||||
use crate::processing::processor::playlist::PlaylistProcessingContext;
|
||||
use crate::processing::processor::{create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags};
|
||||
use crate::ptt::ptt_parse_title;
|
||||
use crate::repository::get_input_storage_path;
|
||||
use crate::repository::persist_input_vod_info;
|
||||
use crate::repository::persist_input_vod_info_batch;
|
||||
use log::{error, info, log_enabled, Level};
|
||||
use crate::repository::{xtream_get_file_path, BPlusTreeQuery};
|
||||
use crate::utils::{trace_if_enabled, debug_if_enabled, xtream};
|
||||
use log::{debug, error, info, log_enabled, trace, warn, Level};
|
||||
use serde_json::Value;
|
||||
use shared::error::TuliproxError;
|
||||
use shared::model::{InputType, PlaylistEntry, StreamProperties, VideoStreamProperties, XtreamVideoInfo};
|
||||
use shared::model::{PlaylistItemType, XtreamCluster};
|
||||
use shared::model::{MediaQuality, PlaylistEntry, PlaylistItem, PlaylistItemType, StreamProperties, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamVideoInfo};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
create_resolve_options_function_for_xtream_target!(vod);
|
||||
|
||||
const BATCH_SIZE: usize = 100;
|
||||
const BATCH_SIZE: usize = 200;
|
||||
|
||||
pub async fn playlist_resolve_vod(app_config: &Arc<AppConfig>,
|
||||
client: &reqwest::Client,
|
||||
target: &ConfigTarget,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
provider_fpl: &mut FetchedPlaylist<'_>,
|
||||
fpl: &mut FetchedPlaylist<'_>) {
|
||||
let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl);
|
||||
if !resolve_movies { return; }
|
||||
#[allow(clippy::too_many_lines)]
|
||||
pub async fn playlist_resolve_vod(
|
||||
ctx: &PlaylistProcessingContext,
|
||||
target: &ConfigTarget,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
provider_fpl: &mut FetchedPlaylist<'_>,
|
||||
fpl: &mut FetchedPlaylist<'_>,
|
||||
) {
|
||||
let resolve_options = get_resolve_vod_options(target, fpl);
|
||||
|
||||
let app_config: &Arc<AppConfig> = &ctx.config;
|
||||
let do_probe = resolve_options.flags.contains(ResolveOptionsFlags::Probe) && app_config.is_ffprobe_enabled().await;
|
||||
|
||||
// Determine if we need to do anything
|
||||
if !resolve_options.flags.contains(ResolveOptionsFlags::Resolve) && !do_probe && !resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing) {
|
||||
return;
|
||||
}
|
||||
|
||||
provider_fpl.source.release_resources(XtreamCluster::Video);
|
||||
|
||||
playlist_resolve_vod_info(
|
||||
ctx,
|
||||
errors,
|
||||
fpl,
|
||||
resolve_options,
|
||||
do_probe,
|
||||
).await;
|
||||
|
||||
if provider_fpl.is_memory() {
|
||||
sync_resolved_vod_properties(provider_fpl, fpl);
|
||||
}
|
||||
|
||||
provider_fpl.source.obtain_resources().await;
|
||||
}
|
||||
|
||||
fn sync_resolved_vod_properties(
|
||||
provider_fpl: &mut FetchedPlaylist<'_>,
|
||||
processed_fpl: &mut FetchedPlaylist<'_>,
|
||||
) {
|
||||
let mut resolved_vod_by_provider_id: HashMap<u32, VideoStreamProperties> = HashMap::new();
|
||||
|
||||
for pli in processed_fpl.items() {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Video
|
||||
|| pli.header.item_type != PlaylistItemType::Video
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(provider_id) = pli.get_provider_id() else {
|
||||
continue;
|
||||
};
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(StreamProperties::Video(properties)) = pli.header.additional_properties.as_ref() {
|
||||
resolved_vod_by_provider_id
|
||||
.entry(provider_id)
|
||||
.or_insert_with(|| properties.as_ref().clone());
|
||||
}
|
||||
}
|
||||
|
||||
if resolved_vod_by_provider_id.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
for source_pli in provider_fpl.items_mut() {
|
||||
if source_pli.header.xtream_cluster != XtreamCluster::Video
|
||||
|| source_pli.header.item_type != PlaylistItemType::Video
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(provider_id) = source_pli.get_provider_id() else {
|
||||
continue;
|
||||
};
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(resolved) = resolved_vod_by_provider_id.get(&provider_id) {
|
||||
source_pli.header.additional_properties =
|
||||
Some(StreamProperties::Video(Box::new(resolved.clone())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines, clippy::too_many_arguments)]
|
||||
async fn playlist_resolve_vod_info(
|
||||
ctx: &PlaylistProcessingContext,
|
||||
_errors: &mut Vec<TuliproxError>,
|
||||
fpl: &mut FetchedPlaylist<'_>,
|
||||
resolve_options: ResolveOptions,
|
||||
do_probe: bool,
|
||||
) {
|
||||
|
||||
let filter = |pli: &PlaylistItem| {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Video
|
||||
|| pli.header.item_type != PlaylistItemType::Video
|
||||
{
|
||||
return false;
|
||||
}
|
||||
true
|
||||
};
|
||||
|
||||
let resolve_tmdb_enabled = fpl.input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
|
||||
|
||||
if resolve_options.flags.contains(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() {
|
||||
queue_background_vod_info(
|
||||
ctx,
|
||||
fpl,
|
||||
filter,
|
||||
&resolve_options,
|
||||
do_probe,
|
||||
resolve_tmdb_enabled,
|
||||
);
|
||||
} else {
|
||||
process_immediate_vod_info(
|
||||
ctx,
|
||||
fpl,
|
||||
filter,
|
||||
resolve_options,
|
||||
do_probe,
|
||||
resolve_tmdb_enabled,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut FetchedPlaylist<'_>,
|
||||
filter: impl Fn(&PlaylistItem) -> bool,
|
||||
resolve_options: ResolveOptions,
|
||||
do_probe: bool,
|
||||
resolve_tmdb_enabled: bool, ) {
|
||||
let input = fpl.input;
|
||||
let working_dir = &app_config.config.load().working_dir;
|
||||
let working_dir = &ctx.config.config.load().working_dir;
|
||||
|
||||
let storage_path = match get_input_storage_path(&input.name, working_dir).await {
|
||||
Ok(storage_path) => storage_path,
|
||||
Ok(path) => path,
|
||||
Err(err) => {
|
||||
error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let vod_info_count = fpl.get_missing_vod_info_count();
|
||||
info!("Found missing {vod_info_count} vod info to resolve");
|
||||
// Keep an optional read query open and reopen lazily only when needed.
|
||||
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Video);
|
||||
let mut db_query_holder = None;
|
||||
let mut _db_lock_holder = None;
|
||||
|
||||
let mut batch: Vec<(ProviderIdType, VideoStreamProperties)> = Vec::with_capacity(BATCH_SIZE);
|
||||
let mut processed_count = 0;
|
||||
let mut last_log_time = Instant::now();
|
||||
let mut processed_vod_info_count = 0;
|
||||
let mut batch = Vec::with_capacity(BATCH_SIZE);
|
||||
let sync_canonical_in_memory = provider_fpl.is_memory();
|
||||
let mut resolved_vod_by_provider_id: Option<HashMap<u32, VideoStreamProperties>> =
|
||||
sync_canonical_in_memory.then(HashMap::new);
|
||||
|
||||
provider_fpl.source.release_resources(XtreamCluster::Video);
|
||||
|
||||
let input = fpl.input;
|
||||
for pli in fpl.items_mut() {
|
||||
if pli.header.xtream_cluster != XtreamCluster::Video
|
||||
|| pli.header.item_type != PlaylistItemType::Video
|
||||
|| pli.has_details() {
|
||||
if !filter(pli) {
|
||||
continue;
|
||||
}
|
||||
let Some(provider_id) = pli.get_provider_id() else { continue; };
|
||||
processed_vod_info_count += 1;
|
||||
if provider_id != 0 {
|
||||
if let Some(content) = playlist_resolve_download_playlist_item(
|
||||
app_config,
|
||||
client,
|
||||
pli,
|
||||
input,
|
||||
errors,
|
||||
resolve_delay,
|
||||
XtreamCluster::Video,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if content.is_empty() { continue; }
|
||||
//tokio::fs::write(&storage_path.join(format!("{provider_id}_vod_info.json")), &content).await.ok();
|
||||
match serde_json::from_str::<XtreamVideoInfo>(&content) {
|
||||
Ok(info) => {
|
||||
let video_stream_props = VideoStreamProperties::from_info(&info, pli);
|
||||
|
||||
batch.push((provider_id, video_stream_props.clone()));
|
||||
if let Some(resolved) = resolved_vod_by_provider_id.as_mut() {
|
||||
resolved.insert(provider_id, video_stream_props.clone());
|
||||
}
|
||||
if batch.len() >= BATCH_SIZE {
|
||||
if let Err(err) = persist_input_vod_info_batch(app_config, &storage_path, XtreamCluster::Video, &input.name, std::mem::take(&mut batch)).await {
|
||||
error!("Failed to persist batch VOD info: {err}");
|
||||
}
|
||||
}
|
||||
let provider_id = if let Ok(uid) = pli.header.id.parse::<u32>() {
|
||||
ProviderIdType::Id(uid)
|
||||
} else {
|
||||
ProviderIdType::from(&*pli.header.id)
|
||||
};
|
||||
|
||||
// This makes the data available for subsequent processing steps like STRM export.
|
||||
pli.header.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props)));
|
||||
let reasons = check_resolve_reasons(&resolve_options, do_probe, resolve_tmdb_enabled, pli);
|
||||
|
||||
if !reasons.is_empty() {
|
||||
// Path 2: Internal Update (Inline)
|
||||
if let Some(prov_mgr) = ctx.provider_manager.as_ref() {
|
||||
if db_query_holder.is_none() && xtream_path.exists() {
|
||||
let lock = ctx.config.file_locks.read_lock(&xtream_path).await;
|
||||
if let Ok(query) = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path) {
|
||||
db_query_holder = Some(query);
|
||||
_db_lock_holder = Some(lock);
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to parse video info for provider {} stream_id {provider_id}: {err} {content}", input.name);
|
||||
}
|
||||
// Pass the optional reference to the query
|
||||
let db_query_ref = db_query_holder.as_mut();
|
||||
|
||||
match update_vod_info_immediate(ctx, prov_mgr, input, pli, provider_id.clone(), &reasons, db_query_ref).await {
|
||||
Ok(Some(updated_props)) => {
|
||||
// Update the current item in fpl
|
||||
pli.header.additional_properties = Some(StreamProperties::Video(Box::new(updated_props.clone())));
|
||||
batch.push((provider_id.clone(), updated_props));
|
||||
if batch.len() >= BATCH_SIZE {
|
||||
// Release lock before persisting to avoid deadlock (persist needs write lock)
|
||||
db_query_holder = None;
|
||||
_db_lock_holder = None;
|
||||
|
||||
// Filter for u32 IDs for persistence
|
||||
let updates: Vec<(u32, VideoStreamProperties)> = batch.iter()
|
||||
.filter_map(|(id, props)| {
|
||||
if let ProviderIdType::Id(vid) = id {
|
||||
Some((*vid, props.clone()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if !updates.is_empty() {
|
||||
if let Err(err) = persist_input_vod_info_batch(
|
||||
&ctx.config,
|
||||
&storage_path,
|
||||
XtreamCluster::Video,
|
||||
&input.name,
|
||||
updates,
|
||||
).await {
|
||||
error!("Failed to persist batch VOD info: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
// Clear batch
|
||||
batch.clear();
|
||||
}
|
||||
|
||||
processed_count += 1;
|
||||
|
||||
// Resolve delay for inline execution to prevent flooding provider
|
||||
// Only delay if we actually did something (reason is not empty) - logic implies we did
|
||||
if resolve_options.resolve_delay > 0 {
|
||||
tokio::time::sleep(Duration::from_secs(u64::from(resolve_options.resolve_delay))).await;
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => {
|
||||
error!("Failed to update VOD metadata for {}: {e}", pli.header.title);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 {
|
||||
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
|
||||
info!("resolved {processed_count} vod info");
|
||||
last_log_time = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Flush the remaining batch if bundled strategy
|
||||
if !batch.is_empty() {
|
||||
if let Err(err) = persist_input_vod_info_batch(app_config, &storage_path, XtreamCluster::Video, &input.name, batch).await {
|
||||
error!("Failed to persist final batch VOD info: {err}");
|
||||
}
|
||||
}
|
||||
// Release lock before final persist
|
||||
_db_lock_holder = None;
|
||||
|
||||
// Keep canonical input playlist updated for subsequent target processing in memory mode.
|
||||
if let Some(resolved_vod_by_provider_id) = resolved_vod_by_provider_id {
|
||||
for source_pli in provider_fpl.items_mut() {
|
||||
if source_pli.header.xtream_cluster != XtreamCluster::Video
|
||||
|| source_pli.header.item_type != PlaylistItemType::Video
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(provider_id) = source_pli.get_provider_id() else { continue; };
|
||||
if provider_id == 0 {
|
||||
continue;
|
||||
}
|
||||
if let Some(resolved) = resolved_vod_by_provider_id.get(&provider_id) {
|
||||
source_pli.header.additional_properties = Some(StreamProperties::Video(Box::new(resolved.clone())));
|
||||
let updates: Vec<(u32, VideoStreamProperties)> = batch.into_iter()
|
||||
.filter_map(|(id, props)| if let ProviderIdType::Id(vid) = id { Some((vid, props)) } else { None })
|
||||
.collect();
|
||||
|
||||
if !updates.is_empty() {
|
||||
if let Err(err) = persist_input_vod_info_batch(&ctx.config, &storage_path, XtreamCluster::Video, &input.name, updates).await {
|
||||
error!("Failed to persist final batch VOD info: {err}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider_fpl.source.obtain_resources().await;
|
||||
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
|
||||
if processed_count > 0 {
|
||||
info!("Processed {processed_count} vod info");
|
||||
}
|
||||
}
|
||||
|
||||
fn check_resolve_reasons(
|
||||
resolve_options: &ResolveOptions,
|
||||
do_probe: bool,
|
||||
resolve_tmdb_enabled: bool,
|
||||
pli: &PlaylistItem,
|
||||
) -> ResolveReasonSet {
|
||||
// Check if we need to do anything for this item
|
||||
let mut reasons = ResolveReasonSet::default();
|
||||
|
||||
let needs_info = check_needs_info(resolve_options, pli, &mut reasons);
|
||||
// TMDB check
|
||||
check_resolve_tmdb(resolve_options, resolve_tmdb_enabled, pli, needs_info, &mut reasons);
|
||||
|
||||
// Probe check
|
||||
if do_probe {
|
||||
check_needs_probe(pli, &mut reasons);
|
||||
}
|
||||
reasons
|
||||
}
|
||||
|
||||
fn check_needs_probe(pli: &PlaylistItem, reasons: &mut ResolveReasonSet) {
|
||||
let needs_probe = match pli.header.additional_properties.as_ref() {
|
||||
Some(StreamProperties::Video(props)) => {
|
||||
let details = props.details.as_ref();
|
||||
let missing_video = !MediaQuality::is_valid_media_info(details.and_then(|d| d.video.as_deref()));
|
||||
let missing_audio = !MediaQuality::is_valid_media_info(details.and_then(|d| d.audio.as_deref()));
|
||||
missing_video || missing_audio
|
||||
}
|
||||
None => {
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if needs_probe {
|
||||
reasons.add(ResolveReason::Probe);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_needs_info(resolve_options: &ResolveOptions, pli: &PlaylistItem, reasons: &mut ResolveReasonSet) -> bool {
|
||||
let needs_info = !pli.has_details() && resolve_options.flags.contains(ResolveOptionsFlags::Resolve);
|
||||
|
||||
if needs_info {
|
||||
reasons.add(ResolveReason::Info);
|
||||
}
|
||||
needs_info
|
||||
}
|
||||
|
||||
fn check_resolve_tmdb(
|
||||
resolve_options: &ResolveOptions,
|
||||
resolve_tmdb_enabled: bool,
|
||||
pli: &PlaylistItem,
|
||||
needs_info: bool,
|
||||
reasons: &mut ResolveReasonSet,
|
||||
) {
|
||||
if resolve_tmdb_enabled && resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing) {
|
||||
if let Some(StreamProperties::Video(video_stream_props)) = pli.header.additional_properties.as_ref() {
|
||||
let has_tmdb = video_stream_props.tmdb.is_some();
|
||||
let has_date = video_stream_props.details.as_ref().and_then(|d| d.release_date.as_ref()).is_some();
|
||||
if !has_tmdb || !has_date {
|
||||
if !has_tmdb { reasons.add(ResolveReason::Tmdb); }
|
||||
if !has_date { reasons.add(ResolveReason::Date); }
|
||||
}
|
||||
} else if !needs_info {
|
||||
// No properties but no download requested? Should be rare
|
||||
reasons.add(ResolveReason::MissingDetails);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn queue_background_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut FetchedPlaylist<'_>,
|
||||
filter: impl Fn(&PlaylistItem) -> bool,
|
||||
resolve_options: &ResolveOptions,
|
||||
do_probe: bool,
|
||||
resolve_tmdb_enabled: bool, ) {
|
||||
let Some(mgr) = ctx.metadata_manager.as_ref() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let input = fpl.input;
|
||||
for pli in fpl.items_mut() {
|
||||
if !filter(pli) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let provider_id = if let Ok(uid) = pli.header.id.parse::<u32>() {
|
||||
ProviderIdType::Id(uid)
|
||||
} else {
|
||||
ProviderIdType::from(&*pli.header.id)
|
||||
};
|
||||
|
||||
let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli);
|
||||
|
||||
if !reasons.is_empty() {
|
||||
let task = UpdateTask::ResolveVod {
|
||||
id: provider_id,
|
||||
reason: reasons,
|
||||
delay: resolve_options.resolve_delay,
|
||||
};
|
||||
mgr.queue_task_background(input.name.clone(), task);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_vod_info_immediate(
|
||||
ctx: &PlaylistProcessingContext,
|
||||
active_provider: &Arc<ActiveProviderManager>,
|
||||
input: &ConfigInput,
|
||||
pli: &PlaylistItem,
|
||||
id: ProviderIdType,
|
||||
reasons: &ResolveReasonSet,
|
||||
db_query: Option<&mut BPlusTreeQuery<u32, XtreamPlaylistItem>>,
|
||||
) -> Result<Option<VideoStreamProperties>, TuliproxError> {
|
||||
let fetch_info = reasons.contains(ResolveReason::Info);
|
||||
|
||||
update_vod_metadata(
|
||||
&ctx.config,
|
||||
&ctx.client,
|
||||
input,
|
||||
id,
|
||||
None, // No active handle, function will acquire one if needed
|
||||
active_provider,
|
||||
Some(&pli.header.title),
|
||||
false,
|
||||
fetch_info,
|
||||
db_query,
|
||||
).await
|
||||
}
|
||||
|
||||
|
||||
/// Updates metadata for a single VOD item (Info + Probe).
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `save` - If true, persists changes to the input database immediately (Instant strategy).
|
||||
/// If false, returns the properties so the caller can batch persist them (Bundled strategy).
|
||||
/// * `fetch_info` - If true, fetches details from Provider API. If false, uses existing/dummy data.
|
||||
/// * `db_query` - Optional pre-opened DB handle to avoid re-opening file.
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub async fn update_vod_metadata(
|
||||
app_config: &Arc<AppConfig>,
|
||||
client: &reqwest::Client,
|
||||
input: &ConfigInput,
|
||||
id: ProviderIdType,
|
||||
active_handle: Option<&ProviderHandle>,
|
||||
active_provider: &Arc<ActiveProviderManager>,
|
||||
playlist_title: Option<&str>,
|
||||
save: bool,
|
||||
fetch_info: bool,
|
||||
db_query: Option<&mut BPlusTreeQuery<u32, XtreamPlaylistItem>>,
|
||||
) -> Result<Option<VideoStreamProperties>, TuliproxError> {
|
||||
let working_dir = &app_config.config.load().working_dir;
|
||||
let storage_path = get_input_storage_path(&input.name, working_dir).await
|
||||
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
|
||||
|
||||
// Check if we should skip based on input options
|
||||
let opts = input.options.as_ref();
|
||||
if opts.is_some_and(|o| o.xtream_skip_vod) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Try to load existing info first to check if we have title/o_name
|
||||
let mut props: Option<VideoStreamProperties> = None;
|
||||
let mut existing_item: Option<XtreamPlaylistItem> = None;
|
||||
|
||||
let stream_id_opt = if let ProviderIdType::Id(vid) = id { Some(vid) } else { None };
|
||||
|
||||
if let Some(stream_id) = stream_id_opt {
|
||||
// Use provided query or open new one
|
||||
if let Some(query) = db_query {
|
||||
if let Ok(Some(item)) = query.query_zero_copy(&stream_id) {
|
||||
existing_item = Some(item.clone());
|
||||
if let Some(StreamProperties::Video(p)) = item.additional_properties.as_ref() {
|
||||
props = Some(*p.clone());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Video);
|
||||
if xtream_path.exists() {
|
||||
let _file_lock = app_config.file_locks.read_lock(&xtream_path).await;
|
||||
if let Ok(mut query) = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path) {
|
||||
if let Ok(Some(item)) = query.query_zero_copy(&stream_id) {
|
||||
existing_item = Some(item.clone());
|
||||
if let Some(StreamProperties::Video(p)) = item.additional_properties.as_ref() {
|
||||
props = Some(*p.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut fetched_new = false;
|
||||
let mut properties_updated = false;
|
||||
|
||||
// Determine the title to use for logging and fallback.
|
||||
// Cloning here to satisfy borrow checker when 'props' is mutated later.
|
||||
let display_title = playlist_title
|
||||
.or_else(|| existing_item.as_ref().map(|i| i.title.as_ref()))
|
||||
.or_else(|| props.as_ref().map(|p| p.name.as_ref()))
|
||||
.unwrap_or("Unknown")
|
||||
.to_string();
|
||||
|
||||
let display_id = stream_id_opt.map_or_else(|| "StringID".to_string(), |v: u32| v.to_string());
|
||||
|
||||
// Input DB is canonical, but if details are missing we still need one info fetch to complete the record.
|
||||
let should_fetch_info = fetch_info && props.as_ref().is_none_or(|p| p.details.is_none());
|
||||
|
||||
// 1. Fetch Info from Provider (only when missing in input DB)
|
||||
if should_fetch_info {
|
||||
if let Some(stream_id) = stream_id_opt {
|
||||
let info_url = xtream::get_xtream_player_api_info_url(input, XtreamCluster::Video, stream_id)
|
||||
.ok_or_else(|| shared::error::info_err!("Failed to build info URL"))?;
|
||||
|
||||
let input_source = InputSource::from(input).with_url(info_url);
|
||||
|
||||
match xtream::get_xtream_stream_info_content(app_config, client, &input_source, true).await {
|
||||
Ok(content) => {
|
||||
if !content.is_empty() {
|
||||
if let Ok(mut json_value) = serde_json::from_str::<Value>(&content) {
|
||||
if let Some(info) = json_value.get_mut("info").and_then(|v| v.as_object_mut()) {
|
||||
crate::model::normalize_release_date(info);
|
||||
}
|
||||
|
||||
if let Ok(info) = serde_json::from_value::<XtreamVideoInfo>(json_value) {
|
||||
props = Some(if let Some(existing) = &existing_item {
|
||||
VideoStreamProperties::from_info(&info, existing)
|
||||
} else {
|
||||
VideoStreamProperties::from_info_without_existing(&info)
|
||||
});
|
||||
fetched_new = true;
|
||||
properties_updated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Failed to fetch VOD info for {display_title} ({display_id}): {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no props yet, create dummy ones if we have enough info (at least a name/title)
|
||||
if props.is_none() {
|
||||
if let Some(title) = playlist_title.or_else(|| existing_item.as_ref().map(|i| i.title.as_ref())) {
|
||||
let mut new_props = VideoStreamProperties {
|
||||
name: title.into(),
|
||||
stream_id: stream_id_opt.unwrap_or(0),
|
||||
container_extension: "".into(), // Will be filled later or by probe
|
||||
..Default::default()
|
||||
};
|
||||
// Ensure details struct exists
|
||||
new_props.details = Some(VideoStreamDetailProperties::default());
|
||||
props = Some(new_props);
|
||||
} else {
|
||||
// We can't proceed without at least a name
|
||||
return Err(shared::error::info_err!("No VOD properties available and no title found for {display_id}"));
|
||||
}
|
||||
}
|
||||
|
||||
// Now it's safe to unwrap because we handled the None case above
|
||||
let mut properties = props.unwrap();
|
||||
|
||||
let resolve_tmdb_enabled = input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
|
||||
|
||||
// 2. Resolve TMDB/Date if missing and explicitly enabled for this input
|
||||
let missing_tmdb = properties.tmdb.is_none();
|
||||
let missing_date = properties.details.as_ref().and_then(|d| d.release_date.as_ref()).is_none();
|
||||
|
||||
if resolve_tmdb_enabled && (missing_tmdb || missing_date) {
|
||||
// Try local parsing first
|
||||
if missing_date && !properties.name.is_empty() {
|
||||
let meta_parse = ptt_parse_title(&properties.name);
|
||||
if let Some(year) = meta_parse.year {
|
||||
if properties.details.is_none() {
|
||||
properties.details = Some(VideoStreamDetailProperties::default());
|
||||
}
|
||||
if let Some(details) = properties.details.as_mut() {
|
||||
details.release_date = Some(format!("{year}-01-01").into());
|
||||
properties_updated = true;
|
||||
debug_if_enabled!("Parsed local year for '{}': {}", properties.name, year);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Re-check missing date after local parse
|
||||
let still_missing_date = properties.details.as_ref().and_then(|d| d.release_date.as_ref()).is_none();
|
||||
|
||||
if missing_tmdb || still_missing_date {
|
||||
let config = app_config.config.load();
|
||||
let library_config = config.library.as_ref();
|
||||
let meta_resolver = MetadataResolver::new(library_config, client.clone());
|
||||
|
||||
let mut meta = None;
|
||||
let mut tried_title = false;
|
||||
|
||||
// 1. & 2. Playlist Title
|
||||
let title_candidate = playlist_title.or_else(|| existing_item.as_ref().map(|i| i.title.as_ref()));
|
||||
if let Some(title) = title_candidate {
|
||||
if !title.is_empty() {
|
||||
debug!("Resolving TMDB for VOD using Playlist Title '{title}' (ID: {display_id})...");
|
||||
meta = meta_resolver.resolve_from_title(title, properties.tmdb, true, resolve_tmdb_enabled).await;
|
||||
tried_title = true;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. API Name (fallback)
|
||||
if (meta.is_none() || (meta.as_ref().is_some_and(|m| m.tmdb_id().is_none())))
|
||||
&& !properties.name.is_empty() {
|
||||
let title_already_tried = if let Some(t) = title_candidate { t == properties.name.as_ref() } else { false };
|
||||
|
||||
if !tried_title || !title_already_tried {
|
||||
trace!("Fallback to API Name '{}'...", properties.name);
|
||||
meta = meta_resolver.resolve_from_title(&properties.name, properties.tmdb, true, resolve_tmdb_enabled).await;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. API Original Name (fallback)
|
||||
if meta.is_none() || (meta.as_ref().is_some_and(|m| m.tmdb_id().is_none())) {
|
||||
if let Some(o_name) = properties.details.as_ref().and_then(|d| d.o_name.as_deref()) {
|
||||
if !o_name.is_empty() && o_name != properties.name.as_ref() {
|
||||
trace!("Fallback to API Original Name '{o_name}'...");
|
||||
meta = meta_resolver.resolve_from_title(o_name, properties.tmdb, true, resolve_tmdb_enabled).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(m) = meta {
|
||||
if properties.tmdb.is_none() {
|
||||
properties.tmdb = m.tmdb_id();
|
||||
properties_updated = true;
|
||||
}
|
||||
if let Some(details) = properties.details.as_mut() {
|
||||
if details.release_date.is_none() {
|
||||
details.release_date = m.year().map(|y| format!("{y}-01-01").into());
|
||||
properties_updated = true;
|
||||
}
|
||||
}
|
||||
if properties_updated {
|
||||
let id_display = properties.tmdb.map_or("None".to_string(), |id| id.to_string());
|
||||
trace_if_enabled!("Resolved TMDB for '{}' (ID: {}): {}", display_title, display_id, id_display);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Probe (if enabled globally in config)
|
||||
let ffprobe_enabled = app_config.is_ffprobe_enabled().await;
|
||||
if ffprobe_enabled {
|
||||
// Ensure details struct exists before probing
|
||||
if properties.details.is_none() {
|
||||
properties.details = Some(VideoStreamDetailProperties::default());
|
||||
}
|
||||
|
||||
let details = properties.details.as_ref().unwrap();
|
||||
let missing_video = !MediaQuality::is_valid_media_info(details.video.as_deref());
|
||||
let missing_audio = !MediaQuality::is_valid_media_info(details.audio.as_deref());
|
||||
|
||||
if missing_video || missing_audio {
|
||||
let input_url = input.url.as_str();
|
||||
let username = input.username.as_deref().unwrap_or("");
|
||||
let password = input.password.as_deref().unwrap_or("");
|
||||
let stream_url = crate::processing::parser::xtream::create_xtream_url(
|
||||
XtreamCluster::Video, input_url, username, password,
|
||||
&StreamProperties::Video(Box::new(properties.clone())),
|
||||
true, true,
|
||||
);
|
||||
|
||||
let config = app_config.config.load();
|
||||
let ffprobe_timeout = config.video.as_ref().and_then(|v| v.ffprobe_timeout).unwrap_or(60);
|
||||
let user_agent = config.default_user_agent.clone();
|
||||
let analyze_duration = 10_000_000;
|
||||
let probe_size = 10_000_000;
|
||||
|
||||
// Acquire Connection logic
|
||||
let temp_handle = if active_handle.is_some() {
|
||||
None // No new handle needed
|
||||
} else {
|
||||
active_provider.acquire_connection_for_probe(&input.name).await
|
||||
};
|
||||
|
||||
if active_handle.is_some() || temp_handle.is_some() {
|
||||
debug_if_enabled!("Probing VOD '{}' (ID: {})", display_title, display_id);
|
||||
if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url(
|
||||
&stream_url,
|
||||
user_agent.as_deref(),
|
||||
analyze_duration,
|
||||
probe_size,
|
||||
ffprobe_timeout,
|
||||
).await {
|
||||
if let Some(details) = properties.details.as_mut() {
|
||||
if let Some(v) = raw_video {
|
||||
details.video = Some(v.to_string().into());
|
||||
properties_updated = true;
|
||||
}
|
||||
if let Some(a) = raw_audio {
|
||||
details.audio = Some(a.to_string().into());
|
||||
properties_updated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(h) = temp_handle {
|
||||
active_provider.release_handle(&h).await;
|
||||
}
|
||||
} else {
|
||||
warn!("Skipping probe for VOD {display_id} due to connection limits");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Persist if updated
|
||||
if properties_updated || fetched_new {
|
||||
if save {
|
||||
if let Some(stream_id) = stream_id_opt {
|
||||
persist_input_vod_info(app_config, &storage_path, XtreamCluster::Video, &input.name, stream_id, &properties)
|
||||
.await
|
||||
.map_err(|e| shared::error::info_err!("Persist error: {e}"))?;
|
||||
}
|
||||
|
||||
debug_if_enabled!("Successfully updated VOD metadata for '{}' (ID: {})", display_title, display_id);
|
||||
}
|
||||
return Ok(Some(properties));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user