euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-17 21:32:17 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

Feature/stream probe (#585)

Browse Source 
Stream probe
This commit is contained in:
euzu
2026-02-14 17:52:35 +01:00
committed by GitHub
parent 9b760a5509
commit 7b8bdaa49b
55 changed files with 2629 additions and 1917 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/processing/parser/xtream.rs
+27 -14
View File
@@ -1,4 +1,4 @@
use crate::model::ConfigInput;
use crate::model::{ConfigInput, ConfigInputFlags};
use crate::model::XtreamCategory;
use crate::utils::request::DynReader;
use crate::utils::xtream::get_xtream_stream_url_base;
@@ -67,8 +67,10 @@ pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStre
series_release_date: Option<&Arc<str>>,
) -> Option<Vec<PlaylistItem>> {
let url = input.url.as_str();
let username = input.username.as_ref().map_or("", |v| v);
let password = input.password.as_ref().map_or("", |v| v);
let (username, password) = (
input.username.as_deref().unwrap_or(""),
input.password.as_deref().unwrap_or(""),
);
if let Some(episodes) = series_info.details.as_ref().and_then(|d| d.episodes.as_ref()) {
let result: Vec<PlaylistItem> = episodes.iter().map(|episode| {
@@ -149,8 +151,10 @@ pub async fn parse_xtream(input: &ConfigInput,
Ok(xtream_categories) => {
let input_name = input.name.clone();
let url = input.url.as_str();
let username = input.username.as_ref().map_or("", |v| v);
let password = input.password.as_ref().map_or("", |v| v);
let (username, password) = (
input.username.as_deref().unwrap_or(""),
input.password.as_deref().unwrap_or(""),
);
match map_to_xtream_streams(xtream_cluster, streams, &input.name).await {
Ok(xtream_streams) => {
@@ -162,8 +166,10 @@ pub async fn parse_xtream(input: &ConfigInput,
channels: vec![],
};
let (live_stream_use_prefix, live_stream_without_extension) = input.options.as_ref()
.map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension));
let (live_stream_use_prefix, live_stream_without_extension) = (
input.has_flag(ConfigInputFlags::XtreamLiveStreamUsePrefix),
input.has_flag(ConfigInputFlags::XtreamLiveStreamWithoutExtension),
);
// Re-implement the loop to add ordinal
let mut ord_counter: u32 = 1;
@@ -234,9 +240,19 @@ where
// 2. Prepare for Stream Parsing
let input_name = input.name.clone();
let url = input.url.as_str().to_string();
let username = input.username.as_ref().map_or("", |v| v).to_string();
let password = input.password.as_ref().map_or("", |v| v).to_string();
let options = input.options.clone();
let (username, password) = (
input.username.as_deref().unwrap_or("").to_string(),
input.password.as_deref().unwrap_or("").to_string(),
);
let (live_stream_use_prefix, live_stream_without_extension) = input.options.as_ref().map_or(
(false, false),
|o| {
(
o.has_flag(ConfigInputFlags::XtreamLiveStreamUsePrefix),
o.has_flag(ConfigInputFlags::XtreamLiveStreamWithoutExtension),
)
},
);
// Map categories for lookup
let group_map: IndexMap<u32, Arc<str>> = xtream_categories.iter().map(|c| (c.category_id, c.category_name.clone())).collect();
@@ -246,9 +262,6 @@ where
let reader = tokio_util::io::SyncIoBridge::new(streams);
let mut deserializer = serde_json::Deserializer::from_reader(reader);
let (live_stream_use_prefix, live_stream_without_extension) = options.as_ref()
.map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension));
let mut source_ordinal = 0u32;
match xtream_cluster {
@@ -438,4 +451,4 @@ mod tests {
serde::Deserializer::deserialize_any(&mut deserializer, visitor).unwrap();
assert_eq!(count, 1);
}
}
}
backend/src/processing/processor/mod.rs
+56 -19
View File
@@ -14,20 +14,31 @@ pub use self::xtream::*;
pub use self::xtream_vod::*;
pub use self::xtream_series::*;
pub use self::stream_probe::*;
use shared::create_bit_set;
use shared::create_bitset;
create_bit_set!(u16, ResolveOptionsFlags, Resolve, TmdbMissing, Probe, Background);
create_bitset!(u8, ResolveOptionsFlags, Resolve, TmdbMissing, Probe, Background);
pub struct ResolveOptions {
pub flags: ResolveOptionsFlagsSet,
flags: ResolveOptionsFlagsSet,
pub resolve_delay: u16,
}
impl ResolveOptions {
#[inline]
pub fn has_flag(&self, flag: ResolveOptionsFlags) -> bool {
self.flags.contains(flag)
}
#[inline]
pub fn unset_flag(&mut self, flag: ResolveOptionsFlags) {
self.flags.unset(flag);
}
}
impl Default for ResolveOptions {
fn default() -> Self {
Self {
flags: ResolveOptionsFlagsSet::from_variants(&[ResolveOptionsFlags::Background]),
resolve_delay: 0,
resolve_delay: default_resolve_delay_secs(),
}
}
}
@@ -41,30 +52,55 @@ macro_rules! create_resolve_options_function_for_xtream_target {
($cluster:ident) => {
paste::paste! {
fn [<get_resolve_ $cluster _options>](target: &ConfigTarget, fpl: &FetchedPlaylist) -> $crate::processing::processor::ResolveOptions {
// Get input options
let (resolve_tmdb_missing, probe_requested) = fpl.input.options.as_ref().map_or_else(||(false, false), |o| (o.resolve_tmdb, o.probe_stream));
match target.get_xtream_output() {
Some(xtream_output) => {
Some(_) => {
let input_options = fpl.input.options.as_ref();
let input_is_xtream = fpl.input.input_type.is_xtream();
let (
resolve_tmdb_missing,
input_resolve_enabled,
input_probe_enabled,
input_resolve_delay,
resolve_background
) = if let Some(options) = input_options {
(
options.has_flag($crate::model::ConfigInputFlags::ResolveTmdb),
options.has_flag($crate::model::ConfigInputFlags::[<Resolve $cluster:camel>]),
options.has_flag($crate::model::ConfigInputFlags::[<Probe $cluster:camel>]),
options.resolve_delay,
options.has_flag($crate::model::ConfigInputFlags::ResolveBackground)
)
} else {
(
false,
false,
false,
shared::utils::default_resolve_delay_secs(),
shared::utils::default_resolve_background(),
)
};
let resolve_enabled = input_resolve_enabled;
let probe_enabled = input_probe_enabled;
let resolve_delay = input_resolve_delay;
let mut flags = $crate::processing::processor::ResolveOptionsFlagsSet::new();
// Map config flags to resolve options flags
if xtream_output.flags.contains($crate::model::XtreamTargetFlags::[<Resolve $cluster:camel>]) && fpl.input.input_type.is_xtream() {
flags.add($crate::processing::processor::ResolveOptionsFlags::Resolve);
if resolve_enabled && input_is_xtream {
flags.set($crate::processing::processor::ResolveOptionsFlags::Resolve);
}
if resolve_tmdb_missing {
flags.add($crate::processing::processor::ResolveOptionsFlags::TmdbMissing);
flags.set($crate::processing::processor::ResolveOptionsFlags::TmdbMissing);
}
if probe_requested
&& fpl.input.input_type.is_xtream()
&& xtream_output.flags.contains($crate::model::XtreamTargetFlags::[<Probe $cluster:camel>]) {
flags.add($crate::processing::processor::ResolveOptionsFlags::Probe);
if input_is_xtream && probe_enabled {
flags.set($crate::processing::processor::ResolveOptionsFlags::Probe);
}
if xtream_output.flags.contains($crate::model::XtreamTargetFlags::ResolveBackground) {
flags.add($crate::processing::processor::ResolveOptionsFlags::Background);
if resolve_background {
flags.set($crate::processing::processor::ResolveOptionsFlags::Background);
}
$crate::processing::processor::ResolveOptions {
flags,
resolve_delay: xtream_output.resolve_delay,
resolve_delay,
}
},
None => $crate::processing::processor::ResolveOptions::default(),
@@ -74,3 +110,4 @@ macro_rules! create_resolve_options_function_for_xtream_target {
};
}
use create_resolve_options_function_for_xtream_target;
use shared::utils::default_resolve_delay_secs;
backend/src/processing/processor/playlist.rs
+275 -192
View File
@@ -1,20 +1,24 @@
use crate::model::{AppConfig, ConfigFavourites, ConfigInput, ConfigRename, MessageContent, ReverseProxyDisabledHeaderConfig, TVGuide};
use crate::utils::{log_memory_snapshot, m3u};
use crate::model::{
AppConfig, ConfigFavourites, ConfigInput, ConfigRename, MessageContent, ReverseProxyDisabledHeaderConfig, TVGuide,
};
use crate::utils::xtream;
use crate::utils::{epg, StepMeasureCallback};
use crate::utils::{log_memory_snapshot, m3u};
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::{Arc, Weak};
use tokio::sync::{Mutex, OwnedRwLockWriteGuard, RwLock};
use tokio::task::JoinSet;
use crate::api::model::{ActiveProviderManager, EventManager, EventMessage, PlaylistStorageState, UpdateGuard, MetadataUpdateManager, ProviderIdType, ResolveReasonSet, ResolveReason, UpdateTask};
use crate::messaging::{send_message};
use crate::api::model::{
ActiveProviderManager, EventManager, EventMessage, MetadataUpdateManager, PlaylistStorageState, ProviderIdType,
ResolveReason, ResolveReasonSet, UpdateGuard, UpdateTask,
};
use crate::messaging::send_message;
use crate::model::FetchedPlaylist;
use crate::model::Mapping;
use crate::model::{ConfigTarget, ProcessTargets, TargetOutput, XtreamTargetFlags};
use crate::model::{ConfigInputFlags, ConfigInputOptions, ConfigTarget, ProcessTargets, TargetOutput};
use crate::processing::input_cache;
use crate::processing::input_cache::ClusterState;
use crate::processing::parser::xmltv::flatten_tvguide;
@@ -34,14 +38,18 @@ use indexmap::IndexMap;
use log::{debug, error, info, log_enabled, warn, Level};
use shared::concat_string;
use shared::error::{get_errors_notify_message, notify_err, TuliproxError};
use shared::foundation::{get_field_value, set_field_value, ValueAccessor, ValueProvider};
use shared::foundation::Filter;
use shared::foundation::{get_field_value, set_field_value, ValueAccessor, ValueProvider};
use shared::model::xtream_const::XTREAM_CLUSTER;
use shared::model::{
CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, PlaylistGroup, PlaylistItem,
PlaylistItemType, PlaylistUpdateState, ProcessingOrder, StreamProperties, XtreamCluster,
};
use shared::model::{InputStats, PlaylistStats, SourceStats, TargetStats, UUIDType};
use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField,
PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState,
ProcessingOrder, StreamProperties, XtreamCluster};
use shared::utils::{create_alias_uuid, default_as_default, interner_gc, Internable};
use shared::utils::{
create_alias_uuid, default_as_default, default_probe_live_interval, default_resolve_delay_secs, interner_gc,
Internable,
};
use std::time::Instant;
const DEFAULT_PROBE_STREAM_DELAY: u16 = 50;
@@ -60,18 +68,24 @@ pub fn apply_filter_to_source(source: &mut dyn PlaylistSource, filter: &Filter)
let cat_id = pli.header.category_id;
let normalized_group = shared::utils::deunicode_string(&group_title).to_lowercase().intern();
let key = (cluster, normalized_group);
groups.entry(key)
groups
.entry(key)
.or_insert_with(|| PlaylistGroup {
id: cat_id,
title: group_title,
channels: vec![],
xtream_cluster: cluster,
})
.channels.push(pli);
.channels
.push(pli);
}
}
if groups.is_empty() { None } else { Some(groups.into_values().collect()) }
if groups.is_empty() {
None
} else {
Some(groups.into_values().collect())
}
}
fn filter_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Option<Vec<PlaylistGroup>> {
@@ -81,8 +95,8 @@ fn filter_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Op
pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) -> Option<Vec<PlaylistGroup>> {
let mut new_playlist = Vec::with_capacity(128);
for pg in playlist.iter_mut() {
let channels = pg.channels.iter()
.filter(|&pli| is_valid(pli, filter, false)).cloned().collect::<Vec<PlaylistItem>>();
let channels =
pg.channels.iter().filter(|&pli| is_valid(pli, filter, false)).cloned().collect::<Vec<PlaylistItem>>();
if !channels.is_empty() {
new_playlist.push(PlaylistGroup {
id: pg.id,
@@ -92,14 +106,16 @@ pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter)
});
}
}
if new_playlist.is_empty() { None } else { Some(new_playlist) }
if new_playlist.is_empty() {
None
} else {
Some(new_playlist)
}
}
fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) {
let assigned_chnos: HashSet<u32> = new_playlist.iter().flat_map(|g| &g.channels)
.filter(|c| c.header.chno != 0)
.map(|c| c.header.chno)
.collect();
let assigned_chnos: HashSet<u32> =
new_playlist.iter().flat_map(|g| &g.channels).filter(|c| c.header.chno != 0).map(|c| c.header.chno).collect();
let mut chno = 1;
for group in new_playlist {
for chan in &mut group.channels {
@@ -150,22 +166,23 @@ fn rename_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Op
let group_title = pli.header.group.clone();
let cluster = pli.header.xtream_cluster;
let cat_id = pli.header.category_id;
groups.entry((cluster, group_title.clone()))
groups
.entry((cluster, group_title.clone()))
.or_insert_with(|| PlaylistGroup {
id: cat_id,
title: group_title,
channels: vec![],
xtream_cluster: cluster,
})
.channels.push(pli);
.channels
.push(pli);
}
Some(groups.into_values().collect())
}
_ => None
_ => None,
}
}
fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, Vec<PlaylistItem>, bool) {
let mut matched = false;
let mut virtual_items = vec![];
@@ -179,7 +196,11 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, V
let provider = ValueProvider { pli: ref_chan, match_as_ascii: mapping.match_as_ascii };
if filter.filter(&provider) {
matched = true;
let mut accessor = ValueAccessor { pli: ref_chan, virtual_items: vec![], match_as_ascii: mapping.match_as_ascii };
let mut accessor = ValueAccessor {
pli: ref_chan,
virtual_items: vec![],
match_as_ascii: mapping.match_as_ascii,
};
script.eval(&mut accessor, templates);
virtual_items.extend(accessor.virtual_items.into_iter().map(|(_, pli)| pli));
}
@@ -214,17 +235,14 @@ fn map_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Optio
for channel in mapped_iter {
let group_title = channel.header.group.clone();
let cluster = channel.header.xtream_cluster;
next_groups.entry((cluster, group_title.clone()))
next_groups
.entry((cluster, group_title.clone()))
.or_insert_with(|| {
grp_id += 1;
PlaylistGroup {
id: grp_id,
title: group_title,
channels: Vec::new(),
xtream_cluster: cluster,
}
PlaylistGroup { id: grp_id, title: group_title, channels: Vec::new(), xtream_cluster: cluster }
})
.channels.push(channel);
.channels
.push(channel);
}
Some(next_groups.into_values().collect())
@@ -249,7 +267,10 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &mut [PlaylistGroup]) {
let new_value = if counter.modifier == CounterModifier::Assign {
padded_cntval
} else {
let value = channel.header.get_field(&counter.field).map_or_else(String::new, |field_value| field_value.to_string());
let value = channel
.header
.get_field(&counter.field)
.map_or_else(String::new, |field_value| field_value.to_string());
if counter.modifier == CounterModifier::Suffix {
format!("{value}{}{padded_cntval}", counter.concat)
} else {
@@ -277,7 +298,7 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo
}
struct PlaylistDownloadResult {
pub downloaded_playlist:Vec<PlaylistGroup>,
pub downloaded_playlist: Vec<PlaylistGroup>,
pub download_err: Vec<TuliproxError>,
pub was_cached: bool,
pub persisted: bool,
@@ -285,21 +306,20 @@ struct PlaylistDownloadResult {
impl PlaylistDownloadResult {
pub fn new(
downloaded_playlist:Vec<PlaylistGroup>,
downloaded_playlist: Vec<PlaylistGroup>,
download_err: Vec<TuliproxError>,
was_cached: bool,
persisted: bool) -> Self {
Self {
downloaded_playlist,
download_err,
was_cached,
persisted,
}
persisted: bool,
) -> Self {
Self { downloaded_playlist, download_err, was_cached, persisted }
}
}
async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc<AppConfig>, input: &ConfigInput) -> PlaylistDownloadResult {
async fn playlist_download_from_input(
client: &reqwest::Client,
app_config: &Arc<AppConfig>,
input: &ConfigInput,
) -> PlaylistDownloadResult {
let config = &*app_config.config.load();
let working_dir = &config.working_dir;
@@ -346,7 +366,9 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc
let (p, e) = m3u::download_m3u_playlist(app_config, client, config, input).await;
(p, e, false)
}
InputType::Xtream => xtream::download_xtream_playlist(app_config, client, input, clusters_to_download.as_deref()).await,
InputType::Xtream => {
xtream::download_xtream_playlist(app_config, client, input, clusters_to_download.as_deref()).await
}
InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![], false),
InputType::Library => {
let (p, e) = library::download_library_playlist(client, app_config, input).await;
@@ -391,7 +413,11 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc
PlaylistDownloadResult::new(playlist, errors, false, persisted)
}
async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (Vec<InputStats>, Vec<TargetStats>, Vec<TuliproxError>) {
#[allow(clippy::too_many_lines)]
async fn process_source(
source_idx: usize,
ctx: &PlaylistProcessingContext,
) -> (Vec<InputStats>, Vec<TargetStats>, Vec<TuliproxError>) {
log_memory_snapshot(format!("source[{source_idx}] start").as_str());
let sources = ctx.config.sources.load();
let mut errors = vec![];
@@ -419,7 +445,10 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (
let (mut download_err, playlist, error) = download_input(ctx, input).await;
if let Some(err) = error {
broadcast_step("Playlist download", &format!("Failed to persist/load input '{}' playlist", input.name));
broadcast_step(
"Playlist download",
&format!("Failed to persist/load input '{}' playlist", input.name),
);
error!("Failed to persist input playlist {}", input.name);
download_err.push(err);
}
@@ -443,30 +472,32 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (
info!("Source is empty {input_name}");
errors.push(notify_err!("Source is empty {input_name}"));
} else {
source_playlists.push(
FetchedPlaylist {
input,
source: playlist_groups,
epg: tvguide,
}
source_playlists.push(FetchedPlaylist { input, source: playlist_groups, epg: tvguide });
log_memory_snapshot(
format!("source[{source_idx}] input '{}' after_source_push", input.name).as_str(),
);
log_memory_snapshot(format!("source[{source_idx}] input '{}' after_source_push", input.name).as_str());
}
let elapsed = start_time.elapsed().as_secs();
input_stats.insert(input_name.clone(), create_input_stat(group_count, channel_count, errors.len(),
input.input_type, input_name, elapsed));
input_stats.insert(
input_name.clone(),
create_input_stat(group_count, channel_count, errors.len(), input.input_type, input_name, elapsed),
);
}
}
if source_downloaded {
if source_playlists.is_empty() {
debug!("Source at index {source_idx} is empty");
errors.push(notify_err!("Source at index {source_idx} is empty: {}", source.inputs.iter().map(Clone::clone).collect::<Vec<Arc<str>>>().join(", ")));
errors.push(notify_err!(
"Source at index {source_idx} is empty: {}",
source.inputs.iter().map(Clone::clone).collect::<Vec<Arc<str>>>().join(", ")
));
} else {
debug_if_enabled!("Source has {} groups", source_playlists.iter_mut().map(FetchedPlaylist::get_channel_count).sum::<usize>());
let enabled_targets: Vec<_> = source.targets
.iter()
.filter(|target| is_target_enabled(target, &ctx.user_targets))
.collect();
debug_if_enabled!(
"Source has {} groups",
source_playlists.iter_mut().map(FetchedPlaylist::get_channel_count).sum::<usize>()
);
let enabled_targets: Vec<_> =
source.targets.iter().filter(|target| is_target_enabled(target, &ctx.user_targets)).collect();
for (idx, target) in enabled_targets.iter().enumerate() {
let consume_input_source = idx + 1 == enabled_targets.len();
@@ -474,9 +505,19 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (
"Processing target '{}' (use_memory_cache={}, consume_input_source={})",
target.name, target.use_memory_cache, consume_input_source
);
log_memory_snapshot(format!("source[{source_idx}] target '{}' before_process", target.name).as_str());
match process_playlist_for_target(ctx, &mut source_playlists, target,
&mut input_stats, &mut errors, consume_input_source).await {
log_memory_snapshot(
format!("source[{source_idx}] target '{}' before_process", target.name).as_str(),
);
match process_playlist_for_target(
ctx,
&mut source_playlists,
target,
&mut input_stats,
&mut errors,
consume_input_source,
)
.await
{
Ok(()) => {
target_stats.push(TargetStats::success(&target.name));
}
@@ -485,7 +526,9 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (
errors.append(&mut err);
}
}
log_memory_snapshot(format!("source[{source_idx}] target '{}' after_process", target.name).as_str());
log_memory_snapshot(
format!("source[{source_idx}] target '{}' after_process", target.name).as_str(),
);
}
}
}
@@ -494,8 +537,11 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (
(input_stats.into_values().collect(), target_stats, errors)
}
async fn download_input_epg(ctx: &PlaylistProcessingContext, input: &Arc<ConfigInput>,
error_list: &mut Vec<TuliproxError>) -> Option<TVGuide> {
async fn download_input_epg(
ctx: &PlaylistProcessingContext,
input: &Arc<ConfigInput>,
error_list: &mut Vec<TuliproxError>,
) -> Option<TVGuide> {
// Download epg for input
let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
let working_dir = &ctx.config.config.load().working_dir;
@@ -507,12 +553,14 @@ async fn download_input_epg(ctx: &PlaylistProcessingContext, input: &Arc<ConfigI
tvguide
}
async fn download_input(ctx: &PlaylistProcessingContext, input: &Arc<ConfigInput>)
-> (Vec<TuliproxError>, Box<dyn PlaylistSource>, Option<TuliproxError>) {
async fn download_input(
ctx: &PlaylistProcessingContext,
input: &Arc<ConfigInput>,
) -> (Vec<TuliproxError>, Box<dyn PlaylistSource>, Option<TuliproxError>) {
// Coordination Logic
let need_download = !ctx.is_input_downloaded(&input.name).await;
let playlist_download_result= if need_download {
let playlist_download_result = if need_download {
// Acquire named lock to prevent thundering herd on same input
let _input_lock = ctx.get_input_lock(&input.name).await;
// Check again after lock
@@ -557,24 +605,24 @@ fn create_broadcast_callback(event_manager: Option<&Arc<EventManager>>) -> StepM
}
}
fn create_input_stat(group_count: usize, channel_count: usize, error_count: usize, input_type: InputType, input_name: &str, secs_took: u64) -> InputStats {
fn create_input_stat(
group_count: usize,
channel_count: usize,
error_count: usize,
input_type: InputType,
input_name: &str,
secs_took: u64,
) -> InputStats {
InputStats {
name: input_name.to_string(),
input_type,
error_count,
raw_stats: PlaylistStats {
group_count,
channel_count,
},
processed_stats: PlaylistStats {
group_count: 0,
channel_count: 0,
},
raw_stats: PlaylistStats { group_count, channel_count },
processed_stats: PlaylistStats { group_count: 0, channel_count: 0 },
secs_took,
}
}
#[derive(Clone)]
pub struct PlaylistProcessingContext {
pub client: reqwest::Client,
@@ -607,13 +655,11 @@ impl PlaylistProcessingContext {
pub async fn get_input_lock(&self, input_name: &Arc<str>) -> OwnedRwLockWriteGuard<()> {
let mut locks = self.input_locks.lock().await;
// Try to upgrade the existing weak reference
let lock = locks.get(input_name)
.and_then(Weak::upgrade)
.unwrap_or_else(|| {
let new_lock = Arc::new(RwLock::new(()));
locks.insert(input_name.clone(), Arc::downgrade(&new_lock));
new_lock
});
let lock = locks.get(input_name).and_then(Weak::upgrade).unwrap_or_else(|| {
let new_lock = Arc::new(RwLock::new(()));
locks.insert(input_name.clone(), Arc::downgrade(&new_lock));
new_lock
});
// Clean up stale references periodically
locks.retain(|_, weak| weak.strong_count() > 0);
@@ -654,8 +700,7 @@ async fn process_sources(processing_ctx: &PlaylistProcessingContext) -> (Vec<Sou
async_tasks.spawn(async move {
// Hold the per-source lock for the full duration of this update.
let current_update_lock = update_lock;
let (input_stats, target_stats, mut res_errors) =
process_source(index, &ctx).await;
let (input_stats, target_stats, mut res_errors) = process_source(index, &ctx).await;
shared_errors.lock().await.append(&mut res_errors);
if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) {
shared_stats.lock().await.push(process_stats);
@@ -663,8 +708,7 @@ async fn process_sources(processing_ctx: &PlaylistProcessingContext) -> (Vec<Sou
drop(current_update_lock);
});
} else {
let (input_stats, target_stats, mut res_errors) =
process_source(index, &ctx).await;
let (input_stats, target_stats, mut res_errors) = process_source(index, &ctx).await;
shared_errors.lock().await.append(&mut res_errors);
if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) {
shared_stats.lock().await.push(process_stats);
@@ -693,12 +737,17 @@ fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe {
ProcessingOrder::Rfm => vec![rename_playlist, filter_playlist, map_playlist],
ProcessingOrder::Rmf => vec![rename_playlist, map_playlist, filter_playlist],
ProcessingOrder::Mfr => vec![map_playlist, filter_playlist, rename_playlist],
ProcessingOrder::Mrf => vec![map_playlist, rename_playlist, filter_playlist]
ProcessingOrder::Mrf => vec![map_playlist, rename_playlist, filter_playlist],
}
}
fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &mut FetchedPlaylist<'a>,
duplicates: &mut HashSet<UUIDType>, consume_source: bool) -> FetchedPlaylist<'a> {
fn execute_pipe<'a>(
target: &ConfigTarget,
pipe: &ProcessingPipe,
fpl: &mut FetchedPlaylist<'a>,
duplicates: &mut HashSet<UUIDType>,
consume_source: bool,
) -> FetchedPlaylist<'a> {
let source = if consume_source {
if fpl.is_memory() {
MemoryPlaylistSource::new(fpl.source.take_groups()).boxed()
@@ -709,11 +758,7 @@ fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &mut Fetc
fpl.clone_source()
};
let mut new_fpl = FetchedPlaylist {
input: fpl.input,
source,
epg: fpl.epg.clone(),
};
let mut new_fpl = FetchedPlaylist { input: fpl.input, source, epg: fpl.epg.clone() };
if target.options.as_ref().is_some_and(|opt| opt.remove_duplicates) {
new_fpl.deduplicate(duplicates);
}
@@ -756,12 +801,13 @@ fn flatten_groups(playlistgroups: Vec<PlaylistGroup>) -> Vec<PlaylistGroup> {
}
#[allow(clippy::too_many_arguments)]
async fn process_playlist_for_target(ctx: &PlaylistProcessingContext,
playlists: &mut [FetchedPlaylist<'_>],
target: &ConfigTarget,
stats: &mut HashMap<Arc<str>, InputStats>,
errors: &mut Vec<TuliproxError>,
consume_input_source: bool,
async fn process_playlist_for_target(
ctx: &PlaylistProcessingContext,
playlists: &mut [FetchedPlaylist<'_>],
target: &ConfigTarget,
stats: &mut HashMap<Arc<str>, InputStats>,
errors: &mut Vec<TuliproxError>,
consume_input_source: bool,
) -> Result<(), Vec<TuliproxError>> {
debug_if_enabled!("Processing order is {}", &target.processing_order);
log_memory_snapshot(format!("target '{}' start", target.name).as_str());
@@ -776,13 +822,19 @@ async fn process_playlist_for_target(ctx: &PlaylistProcessingContext,
let pipe = get_processing_pipe(target);
let mut step = StepMeasure::new(&target.name, broadcast_step);
for provider_fpl in playlists.iter_mut() {
log_memory_snapshot(format!("target '{}' input '{}' before_pipe", target.name, provider_fpl.input.name).as_str());
log_memory_snapshot(
format!("target '{}' input '{}' before_pipe", target.name, provider_fpl.input.name).as_str(),
);
step.broadcast("Executing transformations on '{}' playlist", &target.name);
let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates, consume_input_source);
log_memory_snapshot(format!("target '{}' input '{}' after_pipe", target.name, provider_fpl.input.name).as_str());
log_memory_snapshot(
format!("target '{}' input '{}' after_pipe", target.name, provider_fpl.input.name).as_str(),
);
processed_fpl.sort_by_provider_ordinal();
playlist_resolve(ctx, target, errors, &pipe, provider_fpl, &mut processed_fpl).await;
log_memory_snapshot(format!("target '{}' input '{}' after_vod_resolve", target.name, provider_fpl.input.name).as_str());
log_memory_snapshot(
format!("target '{}' input '{}' after_vod_resolve", target.name, provider_fpl.input.name).as_str(),
);
// stats
let input_entry_name = processed_fpl.input.name.clone();
let group_count = processed_fpl.get_group_count();
@@ -792,9 +844,13 @@ async fn process_playlist_for_target(ctx: &PlaylistProcessingContext,
stat.processed_stats.channel_count = channel_count;
}
process_playlist_epg(&mut processed_fpl, &mut new_epg).await;
log_memory_snapshot(format!("target '{}' input '{}' after_epg_apply", target.name, processed_fpl.input.name).as_str());
log_memory_snapshot(
format!("target '{}' input '{}' after_epg_apply", target.name, processed_fpl.input.name).as_str(),
);
new_playlist.extend(processed_fpl.source.take_groups());
log_memory_snapshot(format!("target '{}' input '{}' after_take_groups", target.name, processed_fpl.input.name).as_str());
log_memory_snapshot(
format!("target '{}' input '{}' after_take_groups", target.name, processed_fpl.input.name).as_str(),
);
tokio::task::yield_now().await;
}
step.tick("filter rename map + epg");
@@ -836,78 +892,103 @@ async fn process_playlist_for_target(ctx: &PlaylistProcessingContext,
step.tick("group watches");
log_memory_snapshot(format!("target '{}' after_group_watches", target.name).as_str());
}
let result = persist_playlist(&ctx.config, &mut flat_new_playlist, flatten_tvguide(new_epg).as_ref(), target, ctx.playlist_state.as_ref()).await;
let result = persist_playlist(
&ctx.config,
&mut flat_new_playlist,
flatten_tvguide(new_epg).as_ref(),
target,
ctx.playlist_state.as_ref(),
)
.await;
step.stop("Persisting playlists");
log_memory_snapshot(format!("target '{}' after_persist", target.name).as_str());
result
}
}
async fn playlist_resolve(ctx: &PlaylistProcessingContext,
target: &ConfigTarget,
errors: &mut Vec<TuliproxError>,
pipe: &ProcessingPipe,
provider_fpl: &mut FetchedPlaylist<'_>,
processed_fpl: &mut FetchedPlaylist<'_>,
async fn playlist_resolve(
ctx: &PlaylistProcessingContext,
target: &ConfigTarget,
errors: &mut Vec<TuliproxError>,
pipe: &ProcessingPipe,
provider_fpl: &mut FetchedPlaylist<'_>,
processed_fpl: &mut FetchedPlaylist<'_>,
) {
playlist_resolve_series(ctx, target, errors, pipe, provider_fpl, processed_fpl).await;
playlist_resolve_vod(ctx, target, errors, provider_fpl, processed_fpl).await;
playlist_probe(ctx, target, processed_fpl).await;
}
fn is_xtream_only_output_target(target: &ConfigTarget) -> bool {
target.output.len() == 1 && matches!(target.output.first(), Some(TargetOutput::Xtream(_)))
fn has_target_m3u_output(target: &ConfigTarget) -> bool {
for output in &target.output {
if matches!(output, TargetOutput::M3u(_)) {
return true;
}
}
false
}
fn get_xtream_cluster_probe_flag(target: &ConfigTarget, cluster: XtreamCluster) -> Option<bool> {
target.get_xtream_output().map(|xtream_output| match cluster {
XtreamCluster::Live => xtream_output.flags.contains(XtreamTargetFlags::ProbeLive),
XtreamCluster::Video => xtream_output.flags.contains(XtreamTargetFlags::ProbeVod),
XtreamCluster::Series => xtream_output.flags.contains(XtreamTargetFlags::ProbeSeries),
fn get_xtream_cluster_probe_flag(
target: &ConfigTarget,
input_options: Option<&ConfigInputOptions>,
cluster: XtreamCluster,
) -> Option<bool> {
target.get_xtream_output().map(|_| match cluster {
XtreamCluster::Live => input_options.is_some_and(|options| options.has_flag(ConfigInputFlags::ProbeLive)),
XtreamCluster::Video => input_options.is_some_and(|options| options.has_flag(ConfigInputFlags::ProbeVod)),
XtreamCluster::Series => input_options.is_some_and(|options| options.has_flag(ConfigInputFlags::ProbeSeries)),
})
}
fn is_cluster_probe_enabled_for_target(target: &ConfigTarget, cluster: XtreamCluster) -> bool {
fn is_cluster_probe_enabled_for_target(
target: &ConfigTarget,
input_options: Option<&ConfigInputOptions>,
cluster: XtreamCluster,
) -> bool {
// Probe flags are only authoritative when the target is exactly one xtream output.
// For mixed outputs (xtream + m3u/strm/...), probing remains enabled because non-xtream outputs have no probe flags.
if !is_xtream_only_output_target(target) {
// For mixed outputs (xtream + m3u), probing remains enabled because non-xtream outputs have no probe flags.
if has_target_m3u_output(target) {
return true;
}
get_xtream_cluster_probe_flag(target, cluster).unwrap_or(true)
get_xtream_cluster_probe_flag(target, input_options, cluster).unwrap_or(true)
}
fn is_probe_supported_item_type(item_type: PlaylistItemType) -> bool {
matches!(
item_type,
PlaylistItemType::Live // we skip other live streams because hls and dash have multiple resolutions
item_type,
PlaylistItemType::Live // we skip other live streams because hls and dash have multiple resolutions
| PlaylistItemType::Video
| PlaylistItemType::LocalVideo
| PlaylistItemType::Series
| PlaylistItemType::LocalSeries
)
)
}
fn has_probe_details(item: &PlaylistItem) -> bool {
match item.header.additional_properties.as_ref() {
Some(StreamProperties::Video(v)) => v
.details
.as_ref()
.is_some_and(|d| d.video.is_some() && d.audio.is_some()),
Some(StreamProperties::Video(v)) => v.details.as_ref().is_some_and(|d| d.video.is_some() && d.audio.is_some()),
Some(StreamProperties::Live(l)) => l.video.is_some() && l.audio.is_some(),
Some(StreamProperties::Episode(e)) => e.video.is_some() && e.audio.is_some(),
Some(StreamProperties::Series(_)) | None => false,
}
}
fn get_live_probe_interval_settings(target: &ConfigTarget, input_type: InputType) -> Option<(u16, u64)> {
if !matches!(input_type, InputType::Xtream | InputType::XtreamBatch) {
fn get_live_probe_interval_settings(
target: &ConfigTarget,
input_type: InputType,
input_options: Option<&ConfigInputOptions>,
) -> Option<(u16, u64)> {
if !(input_type.is_xtream() || input_type.is_m3u()) {
return None;
}
target
.get_xtream_output()
.map(|xtream_output| (xtream_output.resolve_delay, u64::from(xtream_output.probe_live_interval_hours) * 3600))
target.get_xtream_output().map(|_| {
let (resolve_delay, input_probe_live_interval_hours) = input_options.map_or(
(default_resolve_delay_secs(), default_probe_live_interval()),
|options| (options.resolve_delay, options.probe_live_interval_hours),
);
(resolve_delay, u64::from(input_probe_live_interval_hours) * 3600)
})
}
fn needs_live_probe(item: &PlaylistItem, cutoff_ts: i64) -> bool {
@@ -946,7 +1027,7 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
let Some(opts) = fpl.input.options.as_ref() else {
return;
};
if !opts.probe_stream {
if !opts.has_any_flags(ConfigInputFlags::ProbeLive | ConfigInputFlags::ProbeVod | ConfigInputFlags::ProbeSeries) {
return;
}
if !ctx.config.is_ffprobe_enabled().await {
@@ -955,12 +1036,12 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
let input_name = fpl.input.name.clone();
let input_type = fpl.input.input_type;
let xtream_only_output = is_xtream_only_output_target(target);
let live_probe_settings = get_live_probe_interval_settings(target, input_type).map(|(delay, interval_secs)| {
let interval_signed = i64::try_from(interval_secs).unwrap_or(i64::MAX);
let cutoff_ts = chrono::Utc::now().timestamp().saturating_sub(interval_signed);
(delay, interval_secs, cutoff_ts)
});
let live_probe_settings =
get_live_probe_interval_settings(target, input_type, Some(opts)).map(|(delay, interval_secs)| {
let interval_signed = i64::try_from(interval_secs).unwrap_or(i64::MAX);
let cutoff_ts = chrono::Utc::now().timestamp().saturating_sub(interval_signed);
(delay, interval_secs, cutoff_ts)
});
let mut queued_probe_keys: HashSet<(Arc<str>, String)> = HashSet::new();
let mut queued_live_keys: HashSet<ProviderIdType> = HashSet::new();
@@ -968,7 +1049,7 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
let mut queued_stream_count = 0usize;
for item in fpl.items() {
if !is_cluster_probe_enabled_for_target(target, item.header.xtream_cluster) {
if !is_cluster_probe_enabled_for_target(target, Some(opts), item.header.xtream_cluster) {
continue;
}
@@ -996,26 +1077,18 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
}
}
// For xtream-only targets, dedicated xtream resolve/probe processors handle non-live paths.
if xtream_only_output && matches!(input_type, InputType::Xtream | InputType::XtreamBatch) {
continue;
}
if has_probe_details(&item) {
continue;
}
// For M3U, ID is provider id; for Library, ID is UUID.
// For M3U, ID is a provider id; for Library, ID is UUID.
let unique_id = if input_type == InputType::Library {
item.header.uuid.to_valid_uuid()
} else {
item.header.id.to_string()
};
let probe_scope = if item.header.input_name.is_empty() {
input_name.clone()
} else {
item.header.input_name.clone()
};
let probe_scope =
if item.header.input_name.is_empty() { input_name.clone() } else { item.header.input_name.clone() };
if !queued_probe_keys.insert((probe_scope.clone(), unique_id.clone())) {
continue;
@@ -1034,9 +1107,7 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
}
if queued_live_count > 0 || queued_stream_count > 0 {
info!(
"Queued probe tasks for input {input_name} (live_interval={queued_live_count}, generic={queued_stream_count})"
);
info!("Queued probe tasks for input {input_name} (live_interval={queued_live_count}, generic={queued_stream_count})");
}
}
@@ -1046,7 +1117,9 @@ pub fn process_favourites(playlist: &mut Vec<PlaylistGroup>, favourites_cfg: Opt
for pg in playlist.iter() {
for pli in &pg.channels {
// series episodes can't be included in favourites
if pli.header.item_type == PlaylistItemType::Series || pli.header.item_type == PlaylistItemType::LocalSeries {
if pli.header.item_type == PlaylistItemType::Series
|| pli.header.item_type == PlaylistItemType::LocalSeries
{
continue;
}
for fav in favourites {
@@ -1055,10 +1128,7 @@ pub fn process_favourites(playlist: &mut Vec<PlaylistGroup>, favourites_cfg: Opt
channel.header.group.clone_from(&fav.group);
// Update UUID to be an alias of the original
channel.header.uuid = create_alias_uuid(&pli.header.uuid, &fav.group);
fav_groups
.entry((fav.cluster, fav.group.clone()))
.or_default()
.push(channel);
fav_groups.entry((fav.cluster, fav.group.clone())).or_default().push(channel);
}
}
}
@@ -1067,18 +1137,18 @@ pub fn process_favourites(playlist: &mut Vec<PlaylistGroup>, favourites_cfg: Opt
for (fav_group, channels) in fav_groups {
if !channels.is_empty() {
let (xtream_cluster, group_name) = fav_group;
playlist.push(PlaylistGroup {
id: 0,
title: group_name,
channels,
xtream_cluster,
});
playlist.push(PlaylistGroup { id: 0, title: group_name, channels, xtream_cluster });
}
}
}
}
async fn trakt_playlist(client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec<TuliproxError>, playlist: &mut Vec<PlaylistGroup>) -> bool {
async fn trakt_playlist(
client: &reqwest::Client,
target: &ConfigTarget,
errors: &mut Vec<TuliproxError>,
playlist: &mut Vec<PlaylistGroup>,
) -> bool {
match process_trakt_categories_for_target(client, playlist, target).await {
Ok(Some(trakt_categories)) => {
if !trakt_categories.is_empty() {
@@ -1097,7 +1167,12 @@ async fn trakt_playlist(client: &reqwest::Client, target: &ConfigTarget, errors:
true
}
async fn process_watch(app_config: &Arc<AppConfig>, client: &reqwest::Client, target: &ConfigTarget, new_playlist: &[PlaylistGroup]) -> bool {
async fn process_watch(
app_config: &Arc<AppConfig>,
client: &reqwest::Client,
target: &ConfigTarget,
new_playlist: &[PlaylistGroup],
) -> bool {
if let Some(watches) = &target.watch {
if default_as_default().eq_ignore_ascii_case(&target.name) {
error!("can't watch a target with no unique name");
@@ -1108,8 +1183,10 @@ async fn process_watch(app_config: &Arc<AppConfig>, client: &reqwest::Client, ta
new_playlist
.iter()
.filter(|pl| watches.iter().any(|r| r.is_match(&pl.title)))
.map(|pl| process_group_watch(app_config, client, &target.name, pl))
).for_each_concurrent(16, |f| f).await;
.map(|pl| process_group_watch(app_config, client, &target.name, pl)),
)
.for_each_concurrent(16, |f| f)
.await;
true
} else {
@@ -1118,12 +1195,18 @@ async fn process_watch(app_config: &Arc<AppConfig>, client: &reqwest::Client, ta
}
#[allow(clippy::too_many_arguments)]
pub async fn exec_processing(client: &reqwest::Client, app_config: Arc<AppConfig>, targets: Arc<ProcessTargets>,
event_manager: Option<Arc<EventManager>>, playlist_state: Option<Arc<PlaylistStorageState>>,
update_guard: Option<UpdateGuard>, disabled_headers: Option<ReverseProxyDisabledHeaderConfig>,
provider_manager: Option<Arc<ActiveProviderManager>>,
metadata_manager: Option<Arc<MetadataUpdateManager>>) {
let _guard = if let Some(guard) = update_guard {
pub async fn exec_processing(
client: &reqwest::Client,
app_config: Arc<AppConfig>,
targets: Arc<ProcessTargets>,
event_manager: Option<Arc<EventManager>>,
playlist_state: Option<Arc<PlaylistStorageState>>,
update_guard: Option<UpdateGuard>,
disabled_headers: Option<ReverseProxyDisabledHeaderConfig>,
provider_manager: Option<Arc<ActiveProviderManager>>,
metadata_manager: Option<Arc<MetadataUpdateManager>>,
) {
let _guard = if let Some(guard) = update_guard {
if let Some(permit) = guard.try_playlist() {
Some(permit)
} else {
@@ -1184,7 +1267,10 @@ pub async fn exec_processing(client: &reqwest::Client, app_config: Arc<AppConfig
let update_finished_message = format!("🌷 Update process finished! Took {elapsed} secs.");
if let Some(events) = &event_manager {
events.send_event(EventMessage::PlaylistUpdateProgress("Playlist Update".to_string(), update_finished_message.clone()));
events.send_event(EventMessage::PlaylistUpdateProgress(
"Playlist Update".to_string(),
update_finished_message.clone(),
));
}
log_memory_snapshot("exec_processing before_interner_gc");
debug!("StringInterner GC removed {} strings", interner_gc());
@@ -1200,10 +1286,7 @@ mod tests {
use shared::utils::Internable;
fn item_with_props(props: StreamProperties) -> PlaylistItem {
let header = shared::model::PlaylistItemHeader {
additional_properties: Some(props),
..Default::default()
};
let header = shared::model::PlaylistItemHeader { additional_properties: Some(props), ..Default::default() };
PlaylistItem { header }
}
backend/src/processing/processor/xtream.rs
+3 -4
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use shared::error::{TuliproxError};
use crate::model::{AppConfig, ConfigInput};
use crate::model::{AppConfig, ConfigInput, ConfigInputFlags};
use shared::model::{LiveStreamProperties, StreamProperties, XtreamCluster, XtreamPlaylistItem};
use crate::repository::{get_input_storage_path, persist_input_live_info, BPlusTreeQuery, xtream_get_file_path};
use crate::utils::{debug_if_enabled};
@@ -71,9 +71,8 @@ pub async fn update_live_stream_metadata(
let input_url = input.url.as_str();
let username = input.username.as_deref().unwrap_or("");
let password = input.password.as_deref().unwrap_or("");
let opts = input.options.as_ref();
let use_prefix = opts.is_none_or(|o| o.xtream_live_stream_use_prefix);
let no_ext = opts.is_some_and(|o| o.xtream_live_stream_without_extension);
let use_prefix = input.has_flag(ConfigInputFlags::XtreamLiveStreamUsePrefix);
let no_ext = input.has_flag(ConfigInputFlags::XtreamLiveStreamWithoutExtension);
// We generate the URL to probe directly on the provider
let stream_url = create_xtream_url(
backend/src/processing/processor/xtream_series.rs
+36 -32
View File
@@ -3,7 +3,7 @@ use crate::api::model::{ActiveProviderManager, ProviderHandle, ProviderIdType, R
use crate::library::MetadataResolver;
use crate::model::FetchedPlaylist;
use crate::model::{AppConfig, ConfigTarget};
use crate::model::{ConfigInput, InputSource};
use crate::model::{ConfigInput, ConfigInputFlags, InputSource};
use crate::processing::parser::xtream::create_xtream_series_episode_url;
use crate::processing::parser::xtream::parse_xtream_series_info;
use crate::processing::processor::playlist::{PlaylistProcessingContext, ProcessingPipe};
@@ -37,10 +37,16 @@ pub async fn playlist_resolve_series(
provider_fpl: &mut FetchedPlaylist<'_>,
processed_fpl: &mut FetchedPlaylist<'_>,
) {
let resolve_options = get_resolve_series_options(target, processed_fpl);
// Skip-flag is the kill-switch: no resolve, no probe, no iteration.
if processed_fpl.input.has_flag(ConfigInputFlags::XtreamSkipSeries) {
return;
}
let do_probe = resolve_options.flags.contains(ResolveOptionsFlags::Probe) && ctx.config.is_ffprobe_enabled().await;
let mut resolve_options = get_resolve_series_options(target, processed_fpl);
if !ctx.config.is_ffprobe_enabled().await {
resolve_options.unset_flag(ResolveOptionsFlags::Probe);
}
provider_fpl.source.release_resources(XtreamCluster::Series);
playlist_resolve_series_info(
@@ -48,7 +54,6 @@ pub async fn playlist_resolve_series(
errors,
processed_fpl,
resolve_options,
do_probe,
pipe,
target,
)
@@ -66,7 +71,6 @@ async fn playlist_resolve_series_info(ctx: &PlaylistProcessingContext,
_errors: &mut Vec<TuliproxError>,
fpl: &mut FetchedPlaylist<'_>,
resolve_options: ResolveOptions,
do_probe: bool,
pipe: &ProcessingPipe,
target: &ConfigTarget) {
let filter = |pli: &PlaylistItem| {
@@ -79,17 +83,18 @@ async fn playlist_resolve_series_info(ctx: &PlaylistProcessingContext,
};
// Skip if nothing to do
let skip_resolve = !resolve_options.flags.contains(ResolveOptionsFlags::Resolve) && !do_probe && !resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing);
let skip_resolve = !resolve_options.has_flag(ResolveOptionsFlags::Resolve)
&& !resolve_options.has_flag(ResolveOptionsFlags::Probe)
&& !resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing);
let resolve_tmdb_enabled = fpl.input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
let resolve_tmdb_enabled = fpl.input.has_flag(ConfigInputFlags::ResolveTmdb);
let groups_to_add = if resolve_options.flags.contains(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() {
let groups_to_add = if resolve_options.has_flag(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() {
queue_background_series_info(
ctx,
fpl,
filter,
&resolve_options,
do_probe,
skip_resolve,
resolve_tmdb_enabled,
)
@@ -99,7 +104,6 @@ async fn playlist_resolve_series_info(ctx: &PlaylistProcessingContext,
fpl,
filter,
&resolve_options,
do_probe,
skip_resolve,
resolve_tmdb_enabled,
)
@@ -180,7 +184,6 @@ fn queue_background_series_info(
fpl: &mut FetchedPlaylist<'_>,
filter: impl Fn(&PlaylistItem) -> bool,
resolve_options: &ResolveOptions,
do_probe: bool,
skip_resolve: bool,
resolve_tmdb_enabled: bool,
) -> Vec<PlaylistGroup> {
@@ -204,7 +207,7 @@ fn queue_background_series_info(
};
if !skip_resolve {
let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli);
let reasons = check_resolve_reasons(resolve_options, resolve_tmdb_enabled, pli);
if !reasons.is_empty() {
let task = UpdateTask::ResolveSeries {
@@ -229,7 +232,6 @@ async fn process_immediate_series_info(
fpl: &mut FetchedPlaylist<'_>,
filter: impl Fn(&PlaylistItem) -> bool,
resolve_options: &ResolveOptions,
do_probe: bool,
skip_resolve: bool,
resolve_tmdb_enabled: bool,
) -> Vec<PlaylistGroup> {
@@ -265,7 +267,7 @@ async fn process_immediate_series_info(
};
if !skip_resolve {
let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli);
let reasons = check_resolve_reasons(resolve_options, resolve_tmdb_enabled, pli);
if !reasons.is_empty() {
if let Some(active_provider) = ctx.provider_manager.as_ref() {
@@ -420,7 +422,6 @@ async fn update_series_info_immediate(
fn check_resolve_reasons(
resolve_options: &ResolveOptions,
do_probe: bool,
resolve_tmdb_enabled: bool,
pli: &mut PlaylistItem,
) -> ResolveReasonSet {
@@ -429,16 +430,16 @@ fn check_resolve_reasons(
let needs_info = check_needs_info(resolve_options, pli, &mut reasons);
check_resolve_tmdb(resolve_options, resolve_tmdb_enabled, pli, needs_info, &mut reasons);
if do_probe {
check_needs_probe(pli, &mut reasons);
if resolve_options.has_flag(ResolveOptionsFlags::Probe) {
check_needs_probe(pli, needs_info, &mut reasons);
}
reasons
}
fn check_needs_info(resolve_options: &ResolveOptions, pli: &mut PlaylistItem, reasons: &mut ResolveReasonSet) -> bool {
let needs_info = resolve_options.flags.contains(ResolveOptionsFlags::Resolve) && !pli.has_details();
let needs_info = resolve_options.has_flag(ResolveOptionsFlags::Resolve) && !pli.has_details();
if needs_info {
reasons.add(ResolveReason::Info);
reasons.set(ResolveReason::Info);
}
needs_info
}
@@ -450,25 +451,26 @@ fn check_resolve_tmdb(
needs_info: bool,
reasons: &mut ResolveReasonSet,
) {
if resolve_tmdb_enabled && resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing) {
if resolve_tmdb_enabled
&& resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing)
&& needs_info {
if let Some(StreamProperties::Series(series_stream_props)) = pli.header.additional_properties.as_ref() {
let has_tmdb = series_stream_props.tmdb.is_some();
let has_date = series_stream_props.release_date.is_some();
let title_present = !series_stream_props.name.is_empty() || !pli.header.title.is_empty();
if title_present && (!has_tmdb || !has_date) {
if !has_tmdb { reasons.add(ResolveReason::Tmdb); }
if !has_date { reasons.add(ResolveReason::Date); }
if !has_tmdb { reasons.set(ResolveReason::Tmdb); }
if !has_date { reasons.set(ResolveReason::Date); }
}
} else if needs_info {
// Covered by info
} else {
reasons.add(ResolveReason::MissingDetails);
}
}
}
fn check_needs_probe(pli: &mut PlaylistItem, reasons: &mut ResolveReasonSet) {
fn check_needs_probe(pli: &mut PlaylistItem, needs_info: bool, reasons: &mut ResolveReasonSet) {
if !needs_info {
return;
}
if let Some(StreamProperties::Series(series_stream_props)) = pli.header.additional_properties.as_ref() {
if let Some(details) = series_stream_props.details.as_ref() {
if let Some(episodes) = details.episodes.as_ref() {
@@ -476,7 +478,7 @@ fn check_needs_probe(pli: &mut PlaylistItem, reasons: &mut ResolveReasonSet) {
let missing_video = !MediaQuality::is_valid_media_info(ep.video.as_deref());
let missing_audio = !MediaQuality::is_valid_media_info(ep.audio.as_deref());
if missing_video || missing_audio {
reasons.add(ResolveReason::Probe);
reasons.set(ResolveReason::Probe);
break;
}
}
@@ -510,8 +512,7 @@ pub async fn update_series_metadata(
let storage_path = get_input_storage_path(&input.name, working_dir).await
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
let opts = input.options.as_ref();
if opts.is_some_and(|o| o.xtream_skip_series) {
if input.has_flag(ConfigInputFlags::XtreamSkipSeries) {
return Ok(None);
}
@@ -600,10 +601,13 @@ pub async fn update_series_metadata(
let mut properties = props.unwrap();
let resolve_tmdb_enabled = input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
let resolve_tmdb_enabled = input.has_flag(ConfigInputFlags::ResolveTmdb);
// 2. Resolve TMDB/Date if missing
if resolve_tmdb_enabled && (properties.tmdb.is_none() || properties.release_date.is_none()) && !properties.name.is_empty() {
if fetch_info
&& resolve_tmdb_enabled
&& (properties.tmdb.is_none() || properties.release_date.is_none())
&& !properties.name.is_empty() {
let config = app_config.config.load();
let library_config = config.library.as_ref();
let meta_resolver = MetadataResolver::new(library_config, client.clone());
backend/src/processing/processor/xtream_vod.rs
+42 -27
View File
@@ -1,7 +1,7 @@
use crate::api::model::{ActiveProviderManager, ProviderHandle};
use crate::api::model::{ResolveReason, ResolveReasonSet, UpdateTask, ProviderIdType};
use crate::library::MetadataResolver;
use crate::model::ConfigInput;
use crate::model::{ConfigInput, ConfigInputFlags};
use crate::model::FetchedPlaylist;
use crate::model::InputSource;
use crate::model::{AppConfig, ConfigTarget};
@@ -33,13 +33,20 @@ pub async fn playlist_resolve_vod(
provider_fpl: &mut FetchedPlaylist<'_>,
fpl: &mut FetchedPlaylist<'_>,
) {
// Skip-flag is the kill-switch: no resolve, no probe, no iteration.
if fpl.input.has_flag(ConfigInputFlags::XtreamSkipVod) {
return;
}
let resolve_options = get_resolve_vod_options(target, fpl);
let app_config: &Arc<AppConfig> = &ctx.config;
let do_probe = resolve_options.flags.contains(ResolveOptionsFlags::Probe) && app_config.is_ffprobe_enabled().await;
let do_probe = resolve_options.has_flag(ResolveOptionsFlags::Probe) && app_config.is_ffprobe_enabled().await;
// Determine if we need to do anything
if !resolve_options.flags.contains(ResolveOptionsFlags::Resolve) && !do_probe && !resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing) {
if !resolve_options.has_flag(ResolveOptionsFlags::Resolve)
&& !do_probe
&& !resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) {
return;
}
@@ -130,9 +137,9 @@ async fn playlist_resolve_vod_info(
true
};
let resolve_tmdb_enabled = fpl.input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
let resolve_tmdb_enabled = fpl.input.has_flag(ConfigInputFlags::ResolveTmdb);
if resolve_options.flags.contains(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() {
if resolve_options.has_flag(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() {
queue_background_vod_info(
ctx,
fpl,
@@ -227,20 +234,25 @@ async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut F
})
.collect();
if !updates.is_empty() {
if let Err(err) = persist_input_vod_info_batch(
if updates.is_empty() {
batch.clear();
} else {
match persist_input_vod_info_batch(
&ctx.config,
&storage_path,
XtreamCluster::Video,
&input.name,
updates,
).await {
error!("Failed to persist batch VOD info: {err}");
Ok(()) => batch.clear(),
Err(err) => {
error!(
"persist_input_vod_info_batch failed for XtreamCluster::Video on input '{}'. batch.clear() skipped. Error: {err}",
input.name
);
}
}
}
// Clear batch
batch.clear();
}
processed_count += 1;
@@ -301,12 +313,15 @@ fn check_resolve_reasons(
// Probe check
if do_probe {
check_needs_probe(pli, &mut reasons);
check_needs_probe(pli, needs_info, &mut reasons);
}
reasons
}
fn check_needs_probe(pli: &PlaylistItem, reasons: &mut ResolveReasonSet) {
fn check_needs_probe(pli: &PlaylistItem, needs_info: bool, reasons: &mut ResolveReasonSet) {
if !needs_info {
return;
}
let needs_probe = match pli.header.additional_properties.as_ref() {
Some(StreamProperties::Video(props)) => {
let details = props.details.as_ref();
@@ -320,15 +335,15 @@ fn check_needs_probe(pli: &PlaylistItem, reasons: &mut ResolveReasonSet) {
_ => false,
};
if needs_probe {
reasons.add(ResolveReason::Probe);
reasons.set(ResolveReason::Probe);
}
}
fn check_needs_info(resolve_options: &ResolveOptions, pli: &PlaylistItem, reasons: &mut ResolveReasonSet) -> bool {
let needs_info = !pli.has_details() && resolve_options.flags.contains(ResolveOptionsFlags::Resolve);
let needs_info = !pli.has_details() && resolve_options.has_flag(ResolveOptionsFlags::Resolve);
if needs_info {
reasons.add(ResolveReason::Info);
reasons.set(ResolveReason::Info);
}
needs_info
}
@@ -340,17 +355,16 @@ fn check_resolve_tmdb(
needs_info: bool,
reasons: &mut ResolveReasonSet,
) {
if resolve_tmdb_enabled && resolve_options.flags.contains(ResolveOptionsFlags::TmdbMissing) {
if resolve_tmdb_enabled
&& resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing)
&& needs_info {
if let Some(StreamProperties::Video(video_stream_props)) = pli.header.additional_properties.as_ref() {
let has_tmdb = video_stream_props.tmdb.is_some();
let has_date = video_stream_props.details.as_ref().and_then(|d| d.release_date.as_ref()).is_some();
if !has_tmdb || !has_date {
if !has_tmdb { reasons.add(ResolveReason::Tmdb); }
if !has_date { reasons.add(ResolveReason::Date); }
if !has_tmdb { reasons.set(ResolveReason::Tmdb); }
if !has_date { reasons.set(ResolveReason::Date); }
}
} else if !needs_info {
// No properties but no download requested? Should be rare
reasons.add(ResolveReason::MissingDetails);
}
}
}
@@ -410,6 +424,7 @@ async fn update_vod_info_immediate(
Some(&pli.header.title),
false,
fetch_info,
reasons.contains(ResolveReason::Probe),
db_query,
).await
}
@@ -433,6 +448,7 @@ pub async fn update_vod_metadata(
playlist_title: Option<&str>,
save: bool,
fetch_info: bool,
do_probe: bool,
db_query: Option<&mut BPlusTreeQuery<u32, XtreamPlaylistItem>>,
) -> Result<Option<VideoStreamProperties>, TuliproxError> {
let working_dir = &app_config.config.load().working_dir;
@@ -440,8 +456,7 @@ pub async fn update_vod_metadata(
.map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
// Check if we should skip based on input options
let opts = input.options.as_ref();
if opts.is_some_and(|o| o.xtream_skip_vod) {
if input.has_flag(ConfigInputFlags::XtreamSkipVod) {
return Ok(None);
}
@@ -548,13 +563,13 @@ pub async fn update_vod_metadata(
// Now it's safe to unwrap because we handled the None case above
let mut properties = props.unwrap();
let resolve_tmdb_enabled = input.options.as_ref().is_some_and(|o| o.resolve_tmdb);
let resolve_tmdb_enabled = input.has_flag(ConfigInputFlags::ResolveTmdb);
// 2. Resolve TMDB/Date if missing and explicitly enabled for this input
let missing_tmdb = properties.tmdb.is_none();
let missing_date = properties.details.as_ref().and_then(|d| d.release_date.as_ref()).is_none();
if resolve_tmdb_enabled && (missing_tmdb || missing_date) {
if fetch_info && resolve_tmdb_enabled && (missing_tmdb || missing_date) {
// Try local parsing first
if missing_date && !properties.name.is_empty() {
let meta_parse = ptt_parse_title(&properties.name);
@@ -633,7 +648,7 @@ pub async fn update_vod_metadata(
// 3. Probe (if enabled globally in config)
let ffprobe_enabled = app_config.is_ffprobe_enabled().await;
if ffprobe_enabled {
if do_probe && ffprobe_enabled {
// Ensure details struct exists before probing
if properties.details.is_none() {
properties.details = Some(VideoStreamDetailProperties::default());