euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-09-29 04:22:33 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

refactored unwrap calls to make it more stable

Browse Source
This commit is contained in:
euzu
2025-07-22 11:32:46 +02:00
parent 7ef30e1aae
commit 6c238c607f
44 changed files with 560 additions and 479 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/processing/parser/m3u.rs
+38 -28
View File
@@ -1,5 +1,5 @@
use crate::model::{Config, ConfigInput};
use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster, DEFAULT_VIDEO_EXTENSIONS};
use shared::utils::extract_id_from_url;
use std::borrow::BorrowMut;
@@ -116,26 +116,25 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
let mut provider_id = None::<String>;
let mut c = skip_digit(&mut it);
loop {
if c.is_none() {
break;
}
let chr = c.unwrap();
if chr.is_whitespace() {
// skip
} else if chr == ',' {
plih.title = get_value(&mut stack, &mut it);
} else {
stack.push(chr);
let token = token_till(&mut stack, &mut it, '=', true);
if let Some(t) = token {
let value = token_value(&mut stack, &mut it);
let token = t.to_lowercase();
if token.as_str() == "xui-id" {
if !value.is_empty() {
provider_id = Some(value);
}
match c {
None=> break,
Some(chr) => {
if chr.is_whitespace() {
// skip
} else if chr == ',' {
plih.title = get_value(&mut stack, &mut it);
} else {
process_header_fields!(plih, token.as_str(),
stack.push(chr);
let token = token_till(&mut stack, &mut it, '=', true);
if let Some(t) = token {
let value = token_value(&mut stack, &mut it);
let token = t.to_lowercase();
if token.as_str() == "xui-id" {
if !value.is_empty() {
provider_id = Some(value);
}
} else {
process_header_fields!(plih, token.as_str(),
(id, "tvg-id"),
(group, "group-title"),
(name, "tvg-name"),
@@ -146,6 +145,8 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
(logo_small, "tvg-logo-small"),
(time_shift, "timeshift"),
(rec, "tvg-rec"); value);
}
}
}
}
}
@@ -188,7 +189,6 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
plih
}
pub fn consume_m3u<'a, I, F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: I, mut visit: F)
where
I: Iterator<Item=&'a str>,
@@ -197,7 +197,12 @@ where
let mut group: Option<String> = None;
let input_name = input.name.as_str();
let video_suffixes = cfg.video.as_ref().unwrap().extensions.iter().map(String::as_str).collect::<Vec<&str>>();
let video_suffixes = match cfg.video.as_ref() {
Some(config) => {
config.extensions.iter().map(String::as_str).collect::<Vec<&str>>()
},
None => DEFAULT_VIDEO_EXTENSIONS.to_vec()
};
for line in lines {
if line.starts_with("#EXTINF") {
header = Some(String::from(line));
@@ -248,18 +253,23 @@ where
sort_order_idx += 1;
}
std::collections::hash_map::Entry::Occupied(o) => {
sort_order.get_mut(*o.get()).unwrap().push(item);
if let Some(order) = sort_order.get_mut(*o.get()) {
order.push(item);
}
}
}
});
let mut grp_id = 0;
let result: Vec<PlaylistGroup> = sort_order.into_iter().map(|channels| {
let result: Vec<PlaylistGroup> = sort_order.into_iter().filter_map(|channels| {
// create a group based on the first playlist item
let channel = channels.first();
let (cluster, group_title) = channel.map(|pli|
(pli.header.xtream_cluster, &pli.header.group)).unwrap();
grp_id += 1;
PlaylistGroup { id: grp_id, xtream_cluster: cluster, title: group_title.to_string(), channels }
if let Some((cluster, group_title)) = channel.map(|pli|
(pli.header.xtream_cluster, &pli.header.group)) {
grp_id += 1;
Some(PlaylistGroup { id: grp_id, xtream_cluster: cluster, title: group_title.to_string(), channels })
} else {
None
}
}).collect();
result
}
backend/src/processing/parser/xmltv.rs
+38 -25
View File
@@ -2,6 +2,7 @@ use crate::model::{Epg, TVGuide, XmlTag, XmlTagIcon, EPG_ATTRIB_CHANNEL, EPG_ATT
use crate::model::{EpgSmartMatchConfig, PersistedEpgSource};
use crate::processing::processor::epg::EpgIdCache;
use crate::utils::compressed_file_reader::CompressedFileReader;
use dashmap::DashMap;
use deunicode::deunicode;
use quick_xml::events::{BytesStart, BytesText, Event};
use quick_xml::Reader;
@@ -14,7 +15,6 @@ use std::collections::HashMap;
use std::mem;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use dashmap::DashMap;
/// Splits a string at the first delimiter if the prefix matches a known country code.
///
@@ -143,13 +143,14 @@ impl TVGuide {
if !matched && fuzzy_matching {
let (fuzzy_matched, matched_normalized_name) = Self::find_best_fuzzy_match(id_cache, tag);
if fuzzy_matched {
let key = matched_normalized_name.unwrap();
let id = epg_id.to_string();
id_cache.normalized.entry(key).and_modify(|entry| {
entry.replace(id.clone());
id_cache.channel_epg_id.insert(Cow::Owned(id));
matched = true;
});
if let Some(key) = matched_normalized_name {
let id = epg_id.to_string();
id_cache.normalized.entry(key).and_modify(|entry| {
entry.replace(id.clone());
id_cache.channel_epg_id.insert(Cow::Owned(id));
matched = true;
});
}
}
}
matched
@@ -190,9 +191,10 @@ impl TVGuide {
#[allow(clippy::cast_sign_loss)]
let mjw = min(100, (match_jw * 100.0).round() as u16);
if mjw >= match_threshold {
let mut lock = data.lock().unwrap();
if lock.0 < mjw {
*lock = (mjw, Some(Cow::Borrowed(norm_key)));
if let Ok(mut lock) = data.lock() {
if lock.0 < mjw {
*lock = (mjw, Some(Cow::Borrowed(norm_key)));
}
}
if mjw > best_match_threshold {
return true; // (true, matched_normalized_epg_id.map(|s| s.to_string()));
@@ -206,8 +208,9 @@ impl TVGuide {
// is there an early exit strategy ???
if early_exit_flag.load(Ordering::SeqCst) {
let result = data.lock().unwrap().1.take();
return (true, result.as_ref().map(std::string::ToString::to_string));
if let Ok(mut result) = data.lock() {
return (true, result.1.take().as_ref().map(ToString::to_string));
}
}
(false, None)
}
@@ -354,13 +357,14 @@ where
}
}
fn handle_text_tag(stack: &mut [XmlTag], e: &BytesText) {
if !stack.is_empty() {
if let Ok(text) = e.unescape() {
let t = text.trim();
if !t.is_empty() {
stack.last_mut().unwrap().value = Some(t.to_string());
if let Some(tag) = stack.last_mut() {
tag.value = Some(t.to_string());
}
}
}
}
@@ -403,14 +407,16 @@ fn collect_tag_attributes(e: &BytesStart, is_channel: bool, is_program: bool) ->
let attributes = e.attributes().filter_map(Result::ok)
.filter_map(|a| {
let key = String::from_utf8_lossy(a.key.as_ref()).to_string();
let mut value = String::from(a.unescape_value().unwrap().as_ref());
if (is_channel && key == EPG_ATTRIB_ID) || (is_program && key == EPG_ATTRIB_CHANNEL) {
value = value.to_lowercase().to_string();
}
if value.is_empty() {
None
if let Ok(value) = a.unescape_value().as_ref() {
if value.is_empty() {
None
} else if (is_channel && key == EPG_ATTRIB_ID) || (is_program && key == EPG_ATTRIB_CHANNEL) {
Some((key, value.to_lowercase().to_string()))
} else {
Some((key, value.to_string()))
}
} else {
Some((key, value))
None
}
}).collect::<HashMap<String, String>>();
attributes
@@ -466,13 +472,20 @@ pub fn flatten_tvguide(tv_guides: &[Epg]) -> Option<Epg> {
}
});
epg_children.lock().unwrap().extend(children);
if let Ok(mut guard) = epg_children.lock() {
guard.extend(children);
}
});
let children = if let Ok(mut children) = epg_children.lock() {
mem::take(&mut *children)
} else {
vec![]
};
let epg = Epg {
logo_override: false,
priority: 0,
attributes: epg_attributes,
children: mem::take(&mut *epg_children.lock().unwrap()),
children,
};
Some(epg)
}
@@ -492,7 +505,7 @@ mod tests {
/// parse_normalize().unwrap();
/// ```
fn parse_normalize() {
let epg_normalize_dto = EpgSmartMatchConfigDto {..Default::default()};
let epg_normalize_dto = EpgSmartMatchConfigDto { ..Default::default() };
let epg_normalize = EpgSmartMatchConfig::from(epg_normalize_dto);
let normalized = normalize_channel_name("Love Nature", &epg_normalize);
assert_eq!(normalized, "lovenature".to_string());
backend/src/processing/processor/epg.rs
+4 -3
View File
@@ -146,8 +146,9 @@ fn assign_channel_epg(new_epg: &mut Vec<Epg>, fp: &mut FetchedPlaylist, id_cache
for epg_source in epg_sources {
// icon tags
let icon_tags: HashMap<&String, &XmlTag> = epg_source.children.iter()
.filter(|tag| tag.icon != XmlTagIcon::Undefined && tag.get_attribute_value(EPG_ATTRIB_ID).is_some())
.map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t)).collect();
.filter(|tag| tag.icon != XmlTagIcon::Undefined)
.filter_map(|tag| tag.get_attribute_value(EPG_ATTRIB_ID).map(|id| (id, tag)))
.collect();
let assign_values = |chan: &mut PlaylistItem| {
if id_cache.smart_match_enabled && chan.header.epg_channel_id.is_none() {
@@ -169,7 +170,7 @@ fn assign_channel_epg(new_epg: &mut Vec<Epg>, fp: &mut FetchedPlaylist, id_cache
if let Some(epg_channel_id) = chan.header.epg_channel_id.as_ref() {
if !icon_assigned.contains(epg_channel_id) &&
(epg_source.logo_override || chan.header.logo.is_empty() || chan.header.logo_small.is_empty()) {
if let Some(icon_tag) = icon_tags.get(chan.header.epg_channel_id.as_ref().unwrap()) {
if let Some(icon_tag) = icon_tags.get(epg_channel_id) {
if let XmlTagIcon::Src(icon) = &icon_tag.icon {
icon_assigned.insert(epg_channel_id.to_string());
if epg_source.logo_override || chan.header.logo.is_empty() {
backend/src/processing/processor/playlist.rs
+79 -68
View File
@@ -234,70 +234,71 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo
async fn process_source(client: Arc<reqwest::Client>, cfg: Arc<AppConfig>, source_idx: usize, user_targets: Arc<ProcessTargets>) -> (Vec<InputStats>, Vec<TargetStats>, Vec<TuliproxError>) {
let sources = cfg.sources.load();
let source = sources.get_source_at(source_idx).unwrap();
let mut errors = vec![];
let mut input_stats = HashMap::<String, InputStats>::new();
let mut target_stats = Vec::<TargetStats>::new();
let mut source_playlists = Vec::with_capacity(128);
// Download the sources
let mut source_downloaded = false;
for input in &source.inputs {
if is_input_enabled(input, &user_targets) {
let config = cfg.config.load();
let working_dir = &config.working_dir;
if let Some(source) = sources.get_source_at(source_idx) {
let mut source_playlists = Vec::with_capacity(128);
// Download the sources
let mut source_downloaded = false;
for input in &source.inputs {
if is_input_enabled(input, &user_targets) {
let config = cfg.config.load();
let working_dir = &config.working_dir;
source_downloaded = true;
let start_time = Instant::now();
let (mut playlistgroups, mut error_list) = match input.input_type {
InputType::M3u => m3u::get_m3u_playlist(Arc::clone(&client), &config, input, working_dir).await,
InputType::Xtream => xtream::get_xtream_playlist(&config, Arc::clone(&client), input, working_dir).await,
InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![])
};
let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
epg::get_xmltv(Arc::clone(&client), input, working_dir).await
} else {
(None, vec![])
};
errors.append(&mut error_list);
errors.append(&mut tvguide_errors);
let group_count = playlistgroups.len();
let channel_count = playlistgroups.iter()
.map(|group| group.channels.len())
.sum();
let input_name = &input.name;
if playlistgroups.is_empty() {
info!("Source is empty {input_name}");
errors.push(notify_err!(format!("Source is empty {input_name}")));
} else {
playlistgroups.iter_mut().for_each(PlaylistGroup::on_load);
source_playlists.push(
FetchedPlaylist {
input,
playlistgroups,
epg: tvguide,
}
);
}
let elapsed = start_time.elapsed().as_secs();
input_stats.insert(input_name.to_string(), create_input_stat(group_count, channel_count, error_list.len(),
input.input_type, input_name, elapsed));
}
}
if source_downloaded {
if source_playlists.is_empty() {
debug!("Source at index {source_idx} is empty");
errors.push(notify_err!(format!("Source at {source_idx} is empty")));
} else {
debug_if_enabled!("Source has {} groups", source_playlists.iter().map(|fpl| fpl.playlistgroups.len()).sum::<usize>());
for target in &source.targets {
if is_target_enabled(target, &user_targets) {
match process_playlist_for_target(&cfg, Arc::clone(&client), &mut source_playlists, target, &mut input_stats, &mut errors).await {
Ok(()) => {
target_stats.push(TargetStats::success(&target.name));
source_downloaded = true;
let start_time = Instant::now();
let (mut playlistgroups, mut error_list) = match input.input_type {
InputType::M3u => m3u::get_m3u_playlist(Arc::clone(&client), &config, input, working_dir).await,
InputType::Xtream => xtream::get_xtream_playlist(&config, Arc::clone(&client), input, working_dir).await,
InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![])
};
let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
epg::get_xmltv(Arc::clone(&client), input, working_dir).await
} else {
(None, vec![])
};
errors.append(&mut error_list);
errors.append(&mut tvguide_errors);
let group_count = playlistgroups.len();
let channel_count = playlistgroups.iter()
.map(|group| group.channels.len())
.sum();
let input_name = &input.name;
if playlistgroups.is_empty() {
info!("Source is empty {input_name}");
errors.push(notify_err!(format!("Source is empty {input_name}")));
} else {
playlistgroups.iter_mut().for_each(PlaylistGroup::on_load);
source_playlists.push(
FetchedPlaylist {
input,
playlistgroups,
epg: tvguide,
}
Err(mut err) => {
target_stats.push(TargetStats::failure(&target.name));
errors.append(&mut err);
);
}
let elapsed = start_time.elapsed().as_secs();
input_stats.insert(input_name.to_string(), create_input_stat(group_count, channel_count, error_list.len(),
input.input_type, input_name, elapsed));
}
}
if source_downloaded {
if source_playlists.is_empty() {
debug!("Source at index {source_idx} is empty");
errors.push(notify_err!(format!("Source at {source_idx} is empty")));
} else {
debug_if_enabled!("Source has {} groups", source_playlists.iter().map(|fpl| fpl.playlistgroups.len()).sum::<usize>());
for target in &source.targets {
if is_target_enabled(target, &user_targets) {
match process_playlist_for_target(&cfg, Arc::clone(&client), &mut source_playlists, target, &mut input_stats, &mut errors).await {
Ok(()) => {
target_stats.push(TargetStats::success(&target.name));
}
Err(mut err) => {
target_stats.push(TargetStats::failure(&target.name));
errors.append(&mut err);
}
}
}
}
@@ -351,13 +352,17 @@ async fn process_sources(client: Arc<reqwest::Client>, config: &Arc<AppConfig>,
let handles = &mut handle_list;
let process = move || {
// TODO better way ?
let rt = tokio::runtime::Runtime::new().unwrap();
rt.block_on(async {
let (input_stats, target_stats, mut res_errors) = process_source(Arc::clone(&http_client), cfg, index, usr_trgts).await;
shared_errors.lock().await.append(&mut res_errors);
let process_stats = SourceStats::new(input_stats, target_stats);
shared_stats.lock().await.push(process_stats);
});
match tokio::runtime::Runtime::new() {
Ok(rt) => {
rt.block_on(async {
let (input_stats, target_stats, mut res_errors) = process_source(Arc::clone(&http_client), cfg, index, usr_trgts).await;
shared_errors.lock().await.append(&mut res_errors);
let process_stats = SourceStats::new(input_stats, target_stats);
shared_stats.lock().await.push(process_stats);
});
},
Err(err) => error!("Could not create runtime !!! {err}"),
}
};
handles.push(thread::spawn(process));
if handles.len() >= thread_num as usize {
@@ -374,7 +379,11 @@ async fn process_sources(client: Arc<reqwest::Client>, config: &Arc<AppConfig>,
for handle in handle_list {
let _ = handle.join();
}
(Arc::try_unwrap(stats).unwrap().into_inner(), Arc::try_unwrap(errors).unwrap().into_inner())
if let (Ok(s), Ok(e)) = (Arc::try_unwrap(stats), Arc::try_unwrap(errors)) {
(s.into_inner(), e.into_inner())
} else {
(vec![], vec![])
}
}
pub type ProcessingPipe = Vec<fn(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option<Vec<PlaylistGroup>>>;
@@ -430,7 +439,9 @@ fn flatten_groups(playlistgroups: Vec<PlaylistGroup>) -> Vec<PlaylistGroup> {
sort_order.push(group);
}
std::collections::hash_map::Entry::Occupied(o) => {
sort_order.get_mut(*o.get()).unwrap().channels.extend(group.channels);
if let Some(pl_group) = sort_order.get_mut(*o.get()) {
pl_group.channels.extend(group.channels);
}
}
}
}
backend/src/processing/processor/xtream_series.rs
+7 -4
View File
@@ -16,7 +16,7 @@ use std::fs::File;
use std::io::{BufWriter, Write};
use std::sync::Arc;
use std::time::Instant;
use log::{info, log_enabled, Level};
use log::{error, info, log_enabled, Level};
use shared::utils::bincode_serialize;
use crate::model::{XtreamSeriesEpisode, XtreamSeriesInfoEpisode};
use crate::utils;
@@ -36,9 +36,12 @@ fn write_series_episode_record_to_wal_file(
let series_episode = XtreamSeriesEpisode::from(episode);
if let Ok(content_bytes) = bincode_serialize(&series_episode) {
writer.write_all(&provider_id.to_le_bytes())?;
let len = u32::try_from(content_bytes.len()).unwrap();
writer.write_all(&len.to_le_bytes())?;
writer.write_all(&content_bytes)?;
if let Ok(len) = u32::try_from(content_bytes.len()) {
writer.write_all(&len.to_le_bytes())?;
writer.write_all(&content_bytes)?;
} else {
error!("Cant write to WAL file, content length exceeds u32");
}
}
Ok(())
}