euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-30 10:42:29 +01:00
Code Issues Packages Projects Releases 5 Wiki Activity

Merge pull request #469 from euzu/feature/memory_fix

Browse Source 
Feature/memory fix
This commit is contained in:
euzu
2025-12-12 16:54:52 +01:00
committed by GitHub
parent f0ec224f52 12f8326cb3
commit d254ea2b84
59 changed files with 1168 additions and 671 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -76,6 +76,7 @@
- UserTable: Copy credentials to clipboard from user table
- UserTable: Kick user action from streams table
- UserTable: Auto-generated username/password for new proxy users
- Update process uses now streams for data processing.
# 3.1.7 (2025-10-10)
- Added Dark/Bright theme switch
Cargo.lock
Generated
+3 -3
View File
@@ -1096,7 +1096,7 @@ dependencies = [
[[package]]
name = "frontend"
version = "3.2.17"
version = "3.2.18"
dependencies = [
"anyhow",
"base64",
@@ -3793,7 +3793,7 @@ dependencies = [
[[package]]
name = "shared"
version = "3.2.17"
version = "3.2.18"
dependencies = [
"base64",
"bitflags 2.10.0",
@@ -4356,7 +4356,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "tuliprox"
version = "3.2.17"
version = "3.2.18"
dependencies = [
"arc-swap",
"async-compression",
backend/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "tuliprox"
version = "3.2.17"
version = "3.2.18"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -47,7 +47,7 @@ tokio = { version = "1.48", features = ["rt-multi-thread", "parking_lot", "fs"]
#console-subscriber = "0"
#tracing = "0.1"
#tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
tokio-util = { version = "0.7", features = ["io"] }
tokio-util = { version = "0.7", features = ["io", "io-util"] }
tempfile = "3.23"
ruzstd = "0.8"
filetime = "0.2"
backend/src/api/api_utils.rs
+5 -15
View File
@@ -12,7 +12,7 @@ use crate::api::model::{ProviderAllocation, ProviderConfig, ProviderStreamState,
use crate::model::{ConfigInput, ResourceRetryConfig};
use crate::model::{ConfigTarget, ProxyUserCredentials};
use crate::tools::lru_cache::LRUResourceCache;
use crate::utils::create_new_file_for_write;
use crate::utils::{async_file_reader, async_file_writer, create_new_file_for_write};
use crate::utils::request;
use crate::utils::{debug_if_enabled, trace_if_enabled};
use crate::BUILD_TIMESTAMP;
@@ -35,7 +35,6 @@ use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::BufWriter;
use tokio::sync::Mutex;
use url::Url;
@@ -155,7 +154,7 @@ pub async fn serve_file(file_path: &Path, mime_type: mime::Mime) -> impl IntoRes
match tokio::fs::File::open(file_path).await {
Ok(file) => {
let reader = tokio::io::BufReader::new(file);
let reader = async_file_reader(file);
let stream = tokio_util::io::ReaderStream::new(reader);
let body = axum::body::Body::from_stream(stream);
@@ -480,13 +479,7 @@ async fn create_stream_response_details(
| ProviderStreamState::GracePeriod(_provider_name, request_url) => {
let parsed_url = Url::parse(&request_url);
let ((stream, stream_info), reconnect_flag) = if let Ok(url) = parsed_url {
let disabled_headers = app_state
.app_config
.config
.load()
.reverse_proxy
.as_ref()
.and_then(|r| r.disabled_header.clone());
let disabled_headers = app_state.get_disabled_headers();
let provider_stream_factory_options = ProviderStreamFactoryOptions::new(
fingerprint.addr,
item_type,
@@ -1107,7 +1100,7 @@ async fn build_stream_response(
match create_new_file_for_write(&resource_path).await {
Ok(file) => {
debug!("Persisting resource stream {sanitized_resource_url} to {}", resource_path.display());
let writer = BufWriter::new(file);
let writer = async_file_writer(file);
let add_cache_content = get_add_cache_content(resource_url, &app_state.cache);
let tee = tee_stream(byte_stream, writer, &resource_path, add_cache_content);
return try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(tee)));
@@ -1136,10 +1129,7 @@ async fn fetch_resource_with_retry(
.reverse_proxy
.as_ref()
.map_or_else(ResourceRetryConfig::get_default_retry_values, |rp| rp.resource_retry.get_retry_values());
let disabled_headers = config
.reverse_proxy
.as_ref()
.and_then(|r| r.disabled_header.clone());
let disabled_headers = app_state.get_disabled_headers();
for attempt in 0..max_attempts {
let client = request::get_client_request(
&app_state.http_client.load(),
backend/src/api/endpoints/api_playlist_utils.rs
+3 -3
View File
@@ -141,13 +141,13 @@ pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Optio
(axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid Arguments"}))).into_response()
}
pub(in crate::api::endpoints) async fn get_playlist(client: Arc<reqwest::Client>, cfg_input: Option<&Arc<ConfigInput>>, cfg: &Arc<Config>, accept: Option<&String>) -> impl IntoResponse + Send {
pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cfg_input: Option<&Arc<ConfigInput>>, cfg: &Arc<Config>, accept: Option<&String>) -> impl IntoResponse + Send {
match cfg_input {
Some(input) => {
let (result, errors) =
match input.input_type {
InputType::M3u | InputType::M3uBatch => m3u::get_m3u_playlist(&client, cfg, input, &cfg.working_dir).await,
InputType::Xtream | InputType::XtreamBatch => xtream::get_xtream_playlist(cfg, &client, input, &cfg.working_dir).await,
InputType::M3u | InputType::M3uBatch => m3u::get_m3u_playlist(client, cfg, input, &cfg.working_dir).await,
InputType::Xtream | InputType::XtreamBatch => xtream::get_xtream_playlist(cfg, client, input, &cfg.working_dir).await,
};
if result.is_empty() {
let error_strings: Vec<String> = errors.iter().map(std::string::ToString::to_string).collect();
backend/src/api/endpoints/download_api.rs
+13 -3
View File
@@ -1,7 +1,7 @@
use crate::api::model::AppState;
use crate::api::model::{DownloadQueue, FileDownload, FileDownloadRequest};
use crate::model::{AppConfig, VideoDownloadConfig};
use crate::utils::request;
use crate::utils::{async_file_writer, request, IO_BUFFER_SIZE};
use tokio::sync::RwLock;
use futures::stream::TryStreamExt;
use log::info;
@@ -25,15 +25,23 @@ async fn download_file(active: Arc<RwLock<Option<FileDownload>>>, client: &reqwe
if let Some(file_path_str) = file_download.file_path.to_str() {
info!("Downloading {file_path_str}");
match File::create(&file_download.file_path).await {
Ok(mut file) => {
Ok(file) => {
let mut buf_writer = async_file_writer(file);
let mut downloaded: u64 = 0;
let mut stream = response.bytes_stream().map_err(to_io_error);
let mut write_counter = 0;
loop {
match stream.try_next().await {
Ok(item) => {
if let Some(chunk) = item {
match file.write_all(&chunk).await {
match buf_writer.write_all(&chunk).await {
Ok(()) => {
write_counter += chunk.len();
if write_counter >= IO_BUFFER_SIZE {
buf_writer.flush().await.map_err(|err| err.to_string())?;
write_counter = 0;
}
downloaded += chunk.len() as u64;
if let Some(lock) = active.write().await.as_mut() {
lock.size = downloaded;
@@ -47,6 +55,8 @@ async fn download_file(active: Arc<RwLock<Option<FileDownload>>>, client: &reqwe
if let Some(lock) = active.write().await.as_mut() {
lock.size = downloaded;
}
buf_writer.flush().await.map_err(|err| err.to_string())?;
buf_writer.shutdown().await.map_err(|err| err.to_string())?;
return Ok(());
}
}
backend/src/api/endpoints/hls_api.rs
+2 -6
View File
@@ -111,15 +111,11 @@ pub(in crate::api) async fn handle_hls_stream_request(
// Don't forward Range on playlist fetch; segments use original headers in provider path
let filter_header: HeaderFilter = Some(Box::new(|name: &str| !name.eq_ignore_ascii_case("range")));
let forwarded = get_headers_from_request(req_headers, &filter_header);
let config = app_state.app_config.config.load();
let disabled_headers = config
.reverse_proxy
.as_ref()
.and_then(|r| r.disabled_header.clone());
let disabled_headers = app_state.get_disabled_headers();
let headers = request::get_request_headers(None, Some(&forwarded), disabled_headers.as_ref());
let input_source = InputSource::from(input).with_url(request_url);
match request::download_text_content(
Arc::clone(&app_state.http_client.load()),
&app_state.http_client.load(),
disabled_headers.as_ref(),
&input_source,
Some(&headers),
backend/src/api/endpoints/v1_api.rs
+2 -5
View File
@@ -89,11 +89,8 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State<Arc<
method: InputFetchMethod::GET,
headers: HashMap::default(),
};
let disabled_headers = config
.reverse_proxy
.as_ref()
.and_then(|r| r.disabled_header.clone());
return match download_text_content(Arc::clone(&app_state.http_client.load()), disabled_headers.as_ref(), &input_source, None, None).await {
let disabled_headers = app_state.get_disabled_headers();
return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await {
Ok((content, _)) => {
let reader = Cursor::new(content);
let mut geoip = GeoIp::new();
backend/src/api/endpoints/v1_api_config.rs
+2 -6
View File
@@ -118,12 +118,8 @@ async fn config_batch_content(
// The url is changed at this point, we need the raw url for the batch file
if let Some(batch_url) = config_input.t_batch_url.as_ref() {
let input_source = InputSource::from(&*config_input).with_url(batch_url.to_owned());
let config = app_state.app_config.config.load();
let disabled_headers = config
.reverse_proxy
.as_ref()
.and_then(|r| r.disabled_header.clone());
return match download_text_content(Arc::clone(&app_state.http_client.load()), disabled_headers.as_ref(), &input_source, None, None).await {
let disabled_headers = app_state.get_disabled_headers();
return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await {
Ok((content, _path)) => {
// Return CSV with explicit content-type
try_unwrap_body!(axum::response::Response::builder()
backend/src/api/endpoints/v1_api_playlist.rs
+6 -5
View File
@@ -60,14 +60,14 @@ async fn playlist_update(
let process_targets = app_state.app_config.sources.load().validate_targets(user_targets.as_ref());
match process_targets {
Ok(valid_targets) => {
let http_client = Arc::clone(&app_state.http_client.load());
let http_client = app_state.http_client.load().as_ref().clone();
let app_config = Arc::clone(&app_state.app_config);
let event_manager = Arc::clone(&app_state.event_manager);
let playlist_state = Arc::clone(&app_state.playlists);
let valid_targets = Arc::new(valid_targets);
tokio::spawn({
async move {
playlist::exec_processing(http_client, app_config, valid_targets, Some(event_manager), Some(playlist_state)).await;
playlist::exec_processing(&http_client, app_config, valid_targets, Some(event_manager), Some(playlist_state)).await;
}
});
axum::http::StatusCode::ACCEPTED.into_response()
@@ -86,18 +86,19 @@ async fn playlist_content(
axum::extract::Json(playlist_req): axum::extract::Json<PlaylistRequest>,
) -> impl IntoResponse + Send {
let config = app_state.app_config.config.load();
let client = app_state.http_client.load();
match playlist_req {
PlaylistRequest::Target(target_id) => {
get_playlist_for_target(app_state.app_config.get_target_by_id(target_id).as_deref(), &app_state.app_config, accept.as_ref()).await.into_response()
}
PlaylistRequest::Input(input_id) => {
get_playlist(Arc::clone(&app_state.http_client.load()), app_state.app_config.get_input_by_id(input_id).as_ref(), &config, accept.as_ref()).await.into_response()
get_playlist(client.as_ref(), app_state.app_config.get_input_by_id(input_id).as_ref(), &config, accept.as_ref()).await.into_response()
}
PlaylistRequest::CustomXtream(xtream) => {
match Url::parse(&xtream.url) {
Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => {
let input = Arc::new(create_config_input_for_xtream(&xtream.username, &xtream.password, &xtream.url));
get_playlist(Arc::clone(&app_state.http_client.load()), Some(&input), &config, accept.as_ref()).await.into_response()
get_playlist(client.as_ref(), Some(&input), &config, accept.as_ref()).await.into_response()
}
_ => {
(axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response()
@@ -108,7 +109,7 @@ async fn playlist_content(
match Url::parse(&m3u.url) {
Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => {
let input = Arc::new(create_config_input_for_m3u(&m3u.url));
get_playlist(Arc::clone(&app_state.http_client.load()), Some(&input), &config, accept.as_ref()).await.into_response()
get_playlist(client.as_ref(), Some(&input), &config, accept.as_ref()).await.into_response()
}
_ => {
(axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response()
backend/src/api/endpoints/xmltv_api.rs
+3 -3
View File
@@ -8,7 +8,7 @@ use crate::repository::storage::get_target_storage_path;
use crate::repository::storage_const;
use crate::repository::xtream_repository::{xtream_get_epg_file_path, xtream_get_storage_path};
use crate::utils;
use crate::utils::{deobscure_text, obscure_text};
use crate::utils::{async_file_reader, deobscure_text, obscure_text};
use axum::response::IntoResponse;
use chrono::{DateTime, Duration, FixedOffset, NaiveDateTime, Offset, TimeZone, Utc};
use chrono_tz::Tz;
@@ -206,7 +206,7 @@ async fn serve_epg_with_rewrites(
let rewrite_base_url = base_url.to_owned();
match tokio::fs::File::open(epg_path).await {
Ok(file) => {
let reader = tokio::io::BufReader::new(file);
let reader = async_file_reader(file);
let (tx, rx) = tokio::io::duplex(8192);
tokio::spawn(async move {
let mut encoder = async_compression::tokio::write::GzipEncoder::new(tx);
@@ -220,7 +220,7 @@ async fn serve_epg_with_rewrites(
error!("EPG: Failed to write epg doc type {err}");
}
let mut xml_reader = quick_xml::reader::Reader::from_reader(tokio::io::BufReader::new(reader));
let mut xml_reader = quick_xml::reader::Reader::from_reader(async_file_reader(reader));
let mut xml_writer = quick_xml::writer::Writer::new(encoder);
// TODO howto avoid BytesText to escape the doctype "xmltv.dtd" which is written as &quote;xmltv.dtd&quote;
backend/src/api/endpoints/xtream_api.rs
+3 -3
View File
@@ -981,7 +981,7 @@ async fn xtream_get_stream_info_response(
if user.proxy == ProxyType::Redirect && cluster == XtreamCluster::Live {
return redirect(&info_url).into_response();
} else if let Ok(content) = xtream::get_xtream_stream_info(
Arc::clone(&app_state.http_client.load()),
&app_state.http_client.load(),
app_state,
user,
&input,
@@ -1090,7 +1090,7 @@ async fn xtream_get_short_epg(
// TODO serve epg from own db
let input_source = InputSource::from(&*input).with_url(info_url);
return match request::download_text_content(
Arc::clone(&app_state.http_client.load()),
&app_state.http_client.load(),
None,
&input_source,
None,
@@ -1243,7 +1243,7 @@ async fn xtream_get_catchup_response(
let input_source = InputSource::from(&*input).with_url(info_url);
let content = try_result_bad_request!(
xtream::get_xtream_stream_info_content(
Arc::clone(&app_state.http_client.load()),
&app_state.http_client.load(),
&input_source
)
.await
backend/src/api/main_api.rs
+9 -4
View File
@@ -31,6 +31,7 @@ use std::sync::atomic::AtomicI8;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
use tower_governor::key_extractor::SmartIpKeyExtractor;
use tower_http::services::ServeDir;
use crate::api::sys_usage::exec_system_usage;
use crate::repository::storage::get_geoip_path;
use crate::utils::{exec_file_lock_prune, GeoIp};
@@ -114,7 +115,7 @@ async fn create_shared_data(
}
fn exec_update_on_boot(
client: Arc<reqwest::Client>,
client: &reqwest::Client,
app_state: &Arc<AppState>,
targets: &Arc<ProcessTargets>,
) {
@@ -127,8 +128,9 @@ fn exec_update_on_boot(
let app_state_clone = Arc::clone(&app_state.app_config);
let targets_clone = Arc::clone(targets);
let playlist_state = Arc::clone(&app_state.playlists);
let client = client.clone();
tokio::spawn(async move {
playlist::exec_processing(client, app_state_clone, targets_clone, None, Some(playlist_state)).await;
playlist::exec_processing(&client, app_state_clone, targets_clone, None, Some(playlist_state)).await;
});
}
}
@@ -272,15 +274,17 @@ pub async fn start_server(
exec_system_usage(&app_state);
let client = shared_data.http_client.load();
exec_scheduler(
&Arc::clone(&shared_data.http_client.load()),
client.as_ref(),
&app_state,
&targets,
&cancel_token_scheduler,
);
exec_update_on_boot(
Arc::clone(&shared_data.http_client.load()),
client.as_ref(),
&app_state,
&targets,
);
@@ -312,6 +316,7 @@ pub async fn start_server(
// Web Server
let mut router = axum::Router::new()
.route("/healthcheck", axum::routing::get(healthcheck))
.nest_service("/.well-known", ServeDir::new(web_dir_path.join("static/.well-known")))
.merge(ws_api_register(
web_auth_enabled,
web_ui_path.as_str(),
backend/src/api/model/app_state.rs
+11 -4
View File
@@ -2,10 +2,7 @@ use crate::api::config_watch::exec_config_watch;
use crate::api::model::{ActiveProviderManager, ConnectionManager, EventManager, PlaylistStorage, PlaylistStorageState, SharedStreamManager};
use crate::api::model::{ActiveUserManager, DownloadQueue};
use crate::api::scheduler::exec_scheduler;
use crate::model::{
AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets,
ScheduleConfig, SourcesConfig,
};
use crate::model::{AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets, ReverseProxyDisabledHeaderConfig, ScheduleConfig, SourcesConfig};
use crate::repository::playlist_repository::load_target_into_memory_cache;
use crate::tools::lru_cache::LRUResourceCache;
use crate::utils::request::create_client;
@@ -444,6 +441,16 @@ impl AppState {
pub async fn cache_playlist(&self, target_name: &str, playlist: PlaylistStorage) {
self.playlists.cache_playlist(target_name, playlist).await;
}
pub fn get_disabled_headers(&self) -> Option<ReverseProxyDisabledHeaderConfig> {
self
.app_config
.config
.load()
.reverse_proxy
.as_ref()
.and_then(|r| r.disabled_header.clone())
}
}
fn schedules_changed(a: &[ScheduleConfig], b: &[ScheduleConfig]) -> bool {
backend/src/api/model/streams/mod.rs
+3 -2
View File
@@ -5,17 +5,18 @@ mod custom_video_stream;
mod transport_stream_buffer;
// mod chunked_buffer;
mod provider_stream;
mod persist_pipe_stream;
mod provider_stream_factory;
mod shared_stream_manager;
mod active_client_stream;
mod throttled_stream;
pub mod persist_pipe_stream;
pub(in crate) use self::transport_stream_buffer::*;
pub(in crate::api) use self::provider_stream::*;
pub(in crate::api) use self::persist_pipe_stream::*;
pub(in crate::api) use self::provider_stream_factory::*;
pub(in crate::api) use self::shared_stream_manager::*;
pub(in crate::api) use self::active_client_stream::*;
pub(in crate::api) use self::throttled_stream::*;
pub(in crate::api) use self::timed_client_stream::*;
pub(in crate::api) use self::custom_video_stream::*;
pub use self::persist_pipe_stream::*;
backend/src/api/model/streams/persist_pipe_stream.rs
+70 -6
View File
@@ -1,11 +1,13 @@
use std::path::Path;
use crate::api::model::StreamError;
use crate::utils::request::DynReader;
use crate::utils::{async_file_writer, IO_BUFFER_SIZE};
use bytes::Bytes;
use log::{debug, error};
use std::path::{Path,};
use std::sync::Arc;
use tokio::io::AsyncWriteExt;
use tokio_stream::{StreamExt};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio_stream::wrappers::ReceiverStream;
use tokio_stream::StreamExt;
use crate::api::model::StreamError;
pub fn tee_stream<S, W>(
mut stream: S,
@@ -13,8 +15,9 @@ pub fn tee_stream<S, W>(
file_path: &Path,
callback: Arc<dyn Fn(usize) + Send + Sync>,
) -> ReceiverStream<Result<Bytes, StreamError>>
where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin + 'static,
W: tokio::io::AsyncWrite + Send + Unpin + 'static,
where
S: tokio_stream::Stream<Item=Result<Bytes, StreamError>> + Send + Unpin + 'static,
W: tokio::io::AsyncWrite + Send + Unpin + 'static,
{
let (tx, rx) = tokio::sync::mpsc::channel::<Result<Bytes, StreamError>>(32);
let resource_path = file_path.to_owned();
@@ -23,6 +26,7 @@ where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin
let mut total_size = 0usize;
let mut writer_active = true;
let mut write_err: Option<StreamError> = None;
let mut write_counter = 0usize;
while let Some(chunk) = stream.next().await {
match chunk {
@@ -32,6 +36,15 @@ where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin
if let Err(e) = writer.write_all(&bytes).await {
writer_active = false;
write_err = Some(StreamError::StdIo(e.to_string()));
} else {
write_counter += bytes.len();
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
if let Err(err) = writer.flush().await {
writer_active = false;
write_err = Some(StreamError::StdIo(format!("Failed periodic flush of tee_stream writer {err}")));
}
}
}
}
@@ -66,3 +79,54 @@ where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin
ReceiverStream::new(rx)
}
pub async fn tee_dyn_reader(
reader: DynReader,
persist_path: &Path,
callback: Option<Arc<dyn Fn(usize) + Send + Sync>>,
) -> DynReader {
let file = match tokio::fs::File::create(persist_path).await {
Ok(f) => f,
Err(err) => {
error!("Cant open file to write: {}, {err}", persist_path.display());
return reader;
}
};
let (mut tx, rx) = tokio::io::duplex(IO_BUFFER_SIZE);
let mut writer = async_file_writer(file);
let reader_arc = reader;
tokio::spawn(async move {
let mut total_bytes = 0usize;
let mut buf = [0u8; 8192];
let mut reader = reader_arc;
loop {
let n = match reader.read(&mut buf).await {
Ok(0) | Err(_) => break,
Ok(n) => n,
};
total_bytes += n;
if tx.write_all(&buf[..n]).await.is_err() {
break;
}
if writer.write_all(&buf[..n]).await.is_err() {
break;
}
}
let _ = writer.flush().await;
let _ = tx.shutdown().await;
if let Some(cb) = callback {
cb(total_bytes);
}
});
Box::pin(rx) as DynReader
}
backend/src/api/model/streams/provider_stream_factory.rs
+7 -7
View File
@@ -193,7 +193,7 @@ fn get_request_range_start_bytes(req_headers: &HashMap<String, Vec<u8>>) -> Opti
// }
fn prepare_client(
request_client: &Arc<reqwest::Client>,
request_client: &reqwest::Client,
stream_options: &ProviderStreamFactoryOptions,
) -> (reqwest::RequestBuilder, bool) {
let url = stream_options.get_url();
@@ -263,7 +263,7 @@ fn prepare_client(
async fn provider_stream_request(
app_state: &Arc<AppState>,
request_client: &Arc<reqwest::Client>,
request_client: &reqwest::Client,
stream_options: &ProviderStreamFactoryOptions,
) -> Result<Option<ProviderStreamFactoryResponse>, StatusCode> {
let (client, _partial_content) = prepare_client(request_client, stream_options);
@@ -365,7 +365,7 @@ async fn handle_channel_unavailable_stream(app_state: &Arc<AppState>,
async fn get_provider_stream(
app_state: &Arc<AppState>,
client: &Arc<reqwest::Client>,
client: &reqwest::Client,
stream_options: &ProviderStreamFactoryOptions,
) -> Result<Option<ProviderStreamFactoryResponse>, StatusCode> {
let url = stream_options.get_url();
@@ -419,7 +419,7 @@ async fn get_provider_stream(
#[allow(clippy::too_many_lines)]
pub async fn create_provider_stream(
app_state: &Arc<AppState>,
client: &Arc<reqwest::Client>,
client: &reqwest::Client,
stream_options: ProviderStreamFactoryOptions,
) -> Option<ProviderStreamFactoryResponse> {
let client_stream_factory = |stream, reconnect_flag, range_cnt| {
@@ -461,9 +461,9 @@ pub async fn create_provider_stream(
let continue_streaming_signal = continue_client_signal.clone();
let stream_options_provider = stream_options.clone();
let app_state_clone = Arc::clone(app_state);
let client = Arc::clone(client);
let client = client.clone();
let unfold: BoxedProviderStream = stream::unfold((), move |()| {
let client = Arc::clone(&client);
let client = client.clone();
let stream_opts = stream_options_provider.clone();
let continue_streaming = continue_streaming_signal.clone();
let app_state_clone = Arc::clone(&app_state_clone);
@@ -572,7 +572,7 @@ pub async fn create_provider_stream(
// let input = None;
//
// let options = BufferStreamOptions::new(PlaylistItemType::Live, true, true, 0, false);
// let value = create_provider_stream(&cfg, Arc::clone(&client), &url, &req, input, options);
// let value = create_provider_stream(&cfg, &client, &url, &req, input, options);
// let mut values = value.await;
// 'outer: while let Some((ref mut stream, info)) = values.as_mut() {
// if info.is_some() {
backend/src/api/scheduler.rs
+4 -4
View File
@@ -26,7 +26,7 @@ pub fn datetime_to_instant(datetime: DateTime<FixedOffset>) -> Instant {
Instant::now() + duration_until
}
pub fn exec_scheduler(client: &Arc<reqwest::Client>, app_state: &Arc<AppState>, targets: &Arc<ProcessTargets>,
pub fn exec_scheduler(client: &reqwest::Client, app_state: &Arc<AppState>, targets: &Arc<ProcessTargets>,
cancel: &CancellationToken) {
let cfg = &app_state.app_config;
let config = cfg.config.load();
@@ -39,7 +39,7 @@ pub fn exec_scheduler(client: &Arc<reqwest::Client>, app_state: &Arc<AppState>,
let expression = schedule.schedule.clone();
let exec_targets = get_process_targets(cfg, targets, schedule.targets.as_ref());
let app_state_clone = Arc::clone(app_state);
let http_client = Arc::clone(client);
let http_client = client.clone();
let cancel_token = cancel.clone();
tokio::spawn(async move {
start_scheduler(http_client, expression.as_str(), app_state_clone, exec_targets, cancel_token).await;
@@ -47,7 +47,7 @@ pub fn exec_scheduler(client: &Arc<reqwest::Client>, app_state: &Arc<AppState>,
}
}
async fn start_scheduler(client: Arc<reqwest::Client>, expression: &str, app_state: Arc<AppState>,
async fn start_scheduler(client: reqwest::Client, expression: &str, app_state: Arc<AppState>,
targets: Arc<ProcessTargets>, cancel: CancellationToken) {
match Schedule::from_str(expression) {
Ok(schedule) => {
@@ -60,7 +60,7 @@ async fn start_scheduler(client: Arc<reqwest::Client>, expression: &str, app_sta
let app_config = Arc::clone(&app_state.app_config);
let event_manager = Arc::clone(&app_state.event_manager);
let playlist_state = app_state.playlists.clone();
exec_processing(Arc::clone(&client), app_config, Arc::clone(&targets), Some(event_manager), Some(playlist_state)).await;
exec_processing(&client, app_config, Arc::clone(&targets), Some(event_manager), Some(playlist_state)).await;
}
() = cancel.cancelled() => {
break;
backend/src/main.rs
+1 -1
View File
@@ -164,7 +164,7 @@ async fn start_in_cli_mode(cfg: Arc<AppConfig>, targets: Arc<ProcessTargets>) {
error!("Failed to build client {err}");
reqwest::Client::new()
});
playlist::exec_processing(Arc::new(client), cfg, targets, None, None).await;
playlist::exec_processing(&client, cfg, targets, None, None).await;
}
async fn start_in_server_mode(cfg: Arc<AppConfig>, targets: Arc<ProcessTargets>) {
backend/src/messaging.rs
+6 -7
View File
@@ -5,13 +5,12 @@ use log::{debug, error};
use reqwest::header;
use shared::model::MsgKind;
use shared::utils::json_str_to_markdown;
use std::sync::Arc;
fn is_enabled(kind: MsgKind, cfg: &MessagingConfig) -> bool {
cfg.notify_on.contains(&kind)
}
async fn send_http_post_request(client: &Arc<reqwest::Client>, msg: &str, messaging: &MessagingConfig) {
async fn send_http_post_request(client: &reqwest::Client, msg: &str, messaging: &MessagingConfig) {
if let Some(rest) = &messaging.rest {
let data = msg.to_owned();
match client
@@ -27,7 +26,7 @@ async fn send_http_post_request(client: &Arc<reqwest::Client>, msg: &str, messag
}
}
async fn send_telegram_message(client: &Arc<reqwest::Client>, msg: &str, messaging: &MessagingConfig, json: bool) {
async fn send_telegram_message(client: &reqwest::Client, msg: &str, messaging: &MessagingConfig, json: bool) {
// TODO use proxy settings
if let Some(telegram) = &messaging.telegram {
let (message, options) = {
@@ -49,7 +48,7 @@ async fn send_telegram_message(client: &Arc<reqwest::Client>, msg: &str, messagi
}
}
async fn send_pushover_message(client: &Arc<reqwest::Client>, msg: &str, messaging: &MessagingConfig) {
async fn send_pushover_message(client: &reqwest::Client, msg: &str, messaging: &MessagingConfig) {
if let Some(pushover) = &messaging.pushover {
let encoded_message: String = url::form_urlencoded::Serializer::new(String::new())
.append_pair("token", pushover.token.as_str())
@@ -75,7 +74,7 @@ async fn send_pushover_message(client: &Arc<reqwest::Client>, msg: &str, messagi
}
}
async fn dispatch_send_message(client: &Arc<reqwest::Client>, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str, json: bool) {
async fn dispatch_send_message(client: &reqwest::Client, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str, json: bool) {
if let Some(messaging) = cfg {
if is_enabled(kind, messaging) {
tokio::join!(
@@ -87,10 +86,10 @@ async fn dispatch_send_message(client: &Arc<reqwest::Client>, kind: MsgKind, cfg
}
}
pub async fn send_message_json(client: &Arc<reqwest::Client>, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) {
pub async fn send_message_json(client: &reqwest::Client, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) {
dispatch_send_message(client, kind, cfg, msg, true).await;
}
pub async fn send_message(client: &Arc<reqwest::Client>, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) {
pub async fn send_message(client: &reqwest::Client, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) {
dispatch_send_message(client, kind, cfg, msg, false).await;
}
backend/src/model/xmltv.rs
+33 -7
View File
@@ -8,10 +8,12 @@ use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use futures::TryFutureExt;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
use url::Url;
use shared::utils::sanitize_sensitive_info;
use crate::api::model::AppState;
use crate::model::{InputSource};
use crate::utils::async_file_reader;
use crate::utils::request::{get_remote_content_as_stream};
pub const EPG_TAG_TV: &str = "tv";
@@ -93,6 +95,8 @@ impl Epg {
.map(|c| (c, false))
.collect();
let mut write_counter = 0usize;
while let Some((tag, ended)) = stack.pop() {
if ended {
// End-Event
@@ -122,10 +126,19 @@ impl Epg {
}
}
}
write_counter += 1;
if write_counter >= 50 {
writer.get_mut().flush().await?; // flush underlying writer
write_counter = 0;
}
}
// write tv-end
writer.write_event_async(Event::End(BytesEnd::new("tv"))).await?;
let inner = writer.get_mut();
inner.flush().await?;
Ok(())
}
}
@@ -197,11 +210,24 @@ pub async fn parse_xmltv_for_web_ui_from_file(path: &Path) -> Result<EpgTv, Tuli
pub async fn parse_xmltv_for_web_ui_from_url(app_state: &Arc<AppState>, url: &str) -> Result<EpgTv, TuliproxError> {
if let Ok(request_url) = Url::parse(url) {
match get_remote_content_as_stream(
Arc::clone(&app_state.http_client.load()),
&request_url,
InputFetchMethod::GET,
None,
let client = app_state.http_client.load();
let input_source: InputSource = InputSource {
name: String::from("xmltv"),
url: request_url.to_string(),
username: None,
password: None,
method: InputFetchMethod::GET,
headers: HashMap::default(),
};
let disabled_headers = app_state.get_disabled_headers();
match get_remote_content_as_stream(
&client,
&input_source,
None,
&request_url,
disabled_headers.as_ref(),
).await {
Ok((stream, _url)) => {
parse_xmltv_for_web_ui(stream).await
@@ -234,7 +260,7 @@ pub fn get_attr_value(attr: &quick_xml::events::attributes::Attribute) -> Option
#[allow(clippy::too_many_lines)]
async fn parse_xmltv_for_web_ui<R: AsyncRead + Send + Unpin>(reader: R) -> Result<EpgTv, TuliproxError> {
let mut reader = quick_xml::reader::Reader::from_reader(tokio::io::BufReader::new(reader));
let mut reader = quick_xml::reader::Reader::from_reader(async_file_reader(reader));
let mut buf = Vec::new();
let mut channels = Vec::new();
backend/src/model/xtream.rs
+1 -1
View File
@@ -31,7 +31,7 @@ impl XtreamCategory {
}
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, Debug)]
pub struct XtreamStream {
#[serde(default, deserialize_with = "deserialize_as_string")]
pub name: String,
backend/src/processing/parser/m3u.rs
+26 -23
View File
@@ -2,7 +2,8 @@ use crate::model::{Config, ConfigInput};
use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster, DEFAULT_VIDEO_EXTENSIONS};
use shared::utils::extract_id_from_url;
use std::borrow::BorrowMut;
use tokio::io::AsyncBufReadExt;
use crate::utils::request::DynReader;
// other implementations like calculating text_distance on all titles took too much time
// we keep it now as simple as possible and less memory intensive.
@@ -87,9 +88,9 @@ fn skip_digit(it: &mut std::str::Chars) -> Option<char> {
}
}
fn create_empty_playlistitem_header(input_name: &str, url: &str) -> PlaylistItemHeader {
fn create_empty_playlistitem_header(input_name: &str, url: String) -> PlaylistItemHeader {
PlaylistItemHeader {
url: url.to_owned(),
url,
category_id: 0,
input_name: input_name.to_string(),
..Default::default()
@@ -107,7 +108,15 @@ macro_rules! process_header_fields {
};
}
fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url: &str) -> PlaylistItemHeader {
fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url: String) -> PlaylistItemHeader {
let url_id = extract_id_from_url(&url);
let url_types = if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) {
// TODO find Series based on group or configured names
Some((XtreamCluster::Video, PlaylistItemType::Video))
} else {
None
};
let mut plih = create_empty_playlistitem_header(input_name, url);
let mut it = content.chars();
let mut stack = String::with_capacity(64);
@@ -157,7 +166,7 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
plih.epg_channel_id = None;
if let Some(pid) = provider_id {
plih.id = pid;
} else if let Some(chanid) = extract_id_from_url(url) {
} else if let Some(chanid) = url_id {
plih.id = chanid;
}
} else {
@@ -167,11 +176,9 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
}
}
}
if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) {
// TODO find Series based on group or configured names
plih.xtream_cluster = XtreamCluster::Video;
plih.item_type = PlaylistItemType::Video;
if let Some((url_cluster, url_item_type)) = url_types {
plih.xtream_cluster = url_cluster;
plih.item_type = url_item_type;
}
{
@@ -189,10 +196,7 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
plih
}
pub fn consume_m3u<'a, I, F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: I, mut visit: F)
where
I: Iterator<Item=&'a str>,
{
pub async fn consume_m3u<F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: DynReader, mut visit: F) {
let mut header: Option<String> = None;
let mut group: Option<String> = None;
let input_name = input.name.as_str();
@@ -203,9 +207,10 @@ where
},
None => DEFAULT_VIDEO_EXTENSIONS.to_vec()
};
for line in lines {
let mut lines = tokio::io::BufReader::new(lines).lines();
while let Ok(Some(line)) = lines.next_line().await {
if line.starts_with("#EXTINF") {
header = Some(String::from(line));
header = Some(line);
continue;
}
if line.starts_with("#EXTGRP") {
@@ -233,9 +238,7 @@ where
}
}
pub fn parse_m3u<'a, I>(cfg: &Config, input: &ConfigInput, lines: I) -> Vec<PlaylistGroup>
where
I: Iterator<Item=&'a str>,
pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> Vec<PlaylistGroup>
{
let mut sort_order: Vec<Vec<PlaylistItem>> = vec![];
let mut sort_order_idx: usize = 0;
@@ -258,7 +261,7 @@ where
}
}
}
});
}).await;
let mut grp_id = 0;
let result: Vec<PlaylistGroup> = sort_order.into_iter().filter_map(|channels| {
// create a group based on the first playlist item
@@ -285,7 +288,7 @@ mod test {
let url = "http://hello.de/hello.ts";
let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney" , Seven"#;
let pli = process_header(input, &video_suffixes, line, url);
let pli = process_header(input, &video_suffixes, line, url.to_string());
assert_eq!(pli.title, "Seven");
assert_eq!(pli.id, "abc-seven");
assert_eq!(pli.logo, "https://abc.nz/.images/seven.png");
@@ -300,7 +303,7 @@ mod test {
let url = "http://hello.de/hello.ts";
let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney", Seven"#;
let pli = process_header(input, &video_suffixes, line, url);
let pli = process_header(input, &video_suffixes, line, url.to_string());
assert_eq!(pli.title, "Seven");
assert_eq!(pli.id, "abc-seven");
assert_eq!(pli.logo, "https://abc.nz/.images/seven.png");
@@ -315,7 +318,7 @@ mod test {
let url = "http://hello.de/hello.ts";
let line = r#"#EXTINF:-1 tvg-id="abc-seven" xui-id="provider-123" group-title="Sydney", Seven"#;
let pli = process_header(input, &video_suffixes, line, url);
let pli = process_header(input, &video_suffixes, line, url.to_string());
assert_eq!(pli.title, "Seven");
assert_eq!(pli.id, "provider-123"); // Should use xui-id
assert_eq!(pli.epg_channel_id, Some("abc-seven".to_string())); // Should preserve original tvg-id
backend/src/processing/parser/xmltv.rs
+2 -1
View File
@@ -13,6 +13,7 @@ use std::collections::HashMap;
use std::mem;
use std::sync::{Mutex};
use tokio::io::AsyncRead;
use crate::utils::async_file_reader;
use crate::utils::compressed_file_reader_async::CompressedFileReaderAsync;
/// Splits a string at the first delimiter if the prefix matches a known country code.
@@ -412,7 +413,7 @@ where
F: FnMut(XmlTag),
{
let mut stack: Vec<XmlTag> = vec![];
let mut xml_reader = quick_xml::reader::Reader::from_reader(tokio::io::BufReader::new(content));
let mut xml_reader = quick_xml::reader::Reader::from_reader(async_file_reader(content));
let mut buf = Vec::<u8>::new();
loop {
match xml_reader.read_event_into_async(&mut buf).await {
backend/src/processing/parser/xtream.rs
+41 -14
View File
@@ -7,23 +7,31 @@ use crate::utils::xtream::{get_xtream_stream_url_base};
use serde_json::Value;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::task::spawn_blocking;
use crate::utils::request::DynReader;
fn map_to_xtream_category(categories: &Value) -> Result<Vec<XtreamCategory>, TuliproxError> {
match serde_json::from_value::<Vec<XtreamCategory>>(categories.to_owned()) {
Ok(xtream_categories) => Ok(xtream_categories),
Err(err) => {
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to process categories {}", &err)
async fn map_to_xtream_category(categories: DynReader) -> Result<Vec<XtreamCategory>, TuliproxError> {
spawn_blocking(move || {
let reader = tokio_util::io::SyncIoBridge::new(categories);
match serde_json::from_reader::<_, Vec<XtreamCategory>>(reader) {
Ok(xtream_categories) => Ok(xtream_categories),
Err(err) => {
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to process categories {}", &err)
}
}
}
}).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream categories failed: {e}")))?
}
fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: &Value) -> Result<Vec<XtreamStream>, TuliproxError> {
match serde_json::from_value::<Vec<XtreamStream>>(streams.to_owned()) {
async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader) -> Result<Vec<XtreamStream>, TuliproxError> {
spawn_blocking(move || {
let reader = tokio_util::io::SyncIoBridge::new(streams);
match serde_json::from_reader::<_, Vec<XtreamStream>>(reader) {
Ok(stream_list) => Ok(stream_list),
Err(err) => {
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {:?}: {}", xtream_cluster, &err)
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {xtream_cluster}: {err}", )
}
}
}).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream streams failed: {e}")))?
}
fn create_xtream_series_episode_url(url: &str, username: &str, password: &str, episode: &XtreamSeriesInfoEpisode) -> Arc<String> {
@@ -119,18 +127,18 @@ pub fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &st
}
}
pub fn parse_xtream(input: &ConfigInput,
pub async fn parse_xtream(input: &ConfigInput,
xtream_cluster: XtreamCluster,
categories: &Value,
streams: &Value) -> Result<Option<Vec<PlaylistGroup>>, TuliproxError> {
match map_to_xtream_category(categories) {
categories: DynReader,
streams: DynReader) -> Result<Option<Vec<PlaylistGroup>>, TuliproxError> {
match map_to_xtream_category(categories).await {
Ok(xtream_categories) => {
let input_name = input.name.clone();
let url = input.url.as_str();
let username = input.username.as_ref().map_or("", |v| v);
let password = input.password.as_ref().map_or("", |v| v);
match map_to_xtream_streams(xtream_cluster, streams) {
match map_to_xtream_streams(xtream_cluster, streams).await {
Ok(mut xtream_streams) => {
let mut group_map: HashMap<String, XtreamCategory> =
xtream_categories.into_iter().map(|category|
@@ -204,7 +212,10 @@ pub fn parse_xtream(input: &ConfigInput,
#[cfg(test)]
mod tests {
use std::fs;
use shared::model::XtreamCluster;
use crate::model::XtreamSeriesInfo;
use crate::processing::parser::xtream::map_to_xtream_streams;
use crate::utils::async_file_reader;
#[test]
fn test_read_json_file_into_struct() {
@@ -221,4 +232,20 @@ mod tests {
}
#[tokio::test]
async fn test_read_json_stream_into_struct() -> std::io::Result<()> {
let reader = Box::pin(async_file_reader(tokio::fs::File::open("/tmp/vod_streams.json").await?));
match map_to_xtream_streams(XtreamCluster::Video, reader).await {
Ok(_streams) => {
println!("{:?}", _streams.get(1));
println!("{:?}", _streams.get(100));
println!("{:?}", _streams.get(200));
assert!(true);
},
Err(err) => {
assert!(false, "Failed to parse json file: {err}");
}
};
Ok(())
}
}
backend/src/processing/playlist_watch.rs
+2 -3
View File
@@ -1,6 +1,5 @@
use std::collections::BTreeSet;
use std::path::{Path};
use std::sync::Arc;
use log::{error, info};
use shared::model::{MsgKind, PlaylistGroup};
use crate::messaging::{send_message};
@@ -8,7 +7,7 @@ use crate::model::Config;
use crate::utils;
use crate::utils::{bincode_deserialize, bincode_serialize};
pub async fn process_group_watch(client: &Arc<reqwest::Client>, cfg: &Config, target_name: &str, pl: &PlaylistGroup) {
pub async fn process_group_watch(client: &reqwest::Client, cfg: &Config, target_name: &str, pl: &PlaylistGroup) {
let mut new_tree = BTreeSet::new();
pl.channels.iter().for_each(|chan| {
let header = &chan.header;
@@ -60,7 +59,7 @@ struct WatchChanges {
pub removed: Vec<String>,
}
async fn handle_watch_notification(client: &Arc<reqwest::Client>, cfg: &Config, added: &BTreeSet<String>, removed: &BTreeSet<String>, target_name: &str, group_name: &str) {
async fn handle_watch_notification(client: &reqwest::Client, cfg: &Config, added: &BTreeSet<String>, removed: &BTreeSet<String>, target_name: &str, group_name: &str) {
let added = added.iter().map(std::string::ToString::to_string).collect::<Vec<String>>();
let removed = removed.iter().map(std::string::ToString::to_string).collect::<Vec<String>>();
if !added.is_empty() || !removed.is_empty() {
backend/src/processing/processor/playlist.rs
+24 -25
View File
@@ -24,12 +24,11 @@ use crate::processing::processor::trakt::process_trakt_categories_for_target;
use crate::processing::processor::xtream_series::playlist_resolve_series;
use crate::processing::processor::xtream_vod::playlist_resolve_vod;
use crate::repository::playlist_repository::persist_playlist;
use crate::utils::debug_if_enabled;
use crate::utils::{debug_if_enabled, trace_if_enabled};
use crate::utils::StepMeasure;
use deunicode::deunicode;
use futures::StreamExt;
use log::{debug, error, info, log_enabled, trace, warn, Level};
use reqwest::Client;
use shared::error::{get_errors_notify_message, notify_err, TuliproxError};
use shared::foundation::filter::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider};
use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster};
@@ -135,7 +134,7 @@ fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec<ConfigRename>>) {
let value = get_field_value(result, r.field);
let cap = r.pattern.replace_all(value.as_str(), &r.new_name);
if log_enabled!(log::Level::Debug) && *value != cap {
debug_if_enabled!("Renamed {}={} to {}", &r.field, value, cap);
trace_if_enabled!("Renamed {}={value} to {cap}", &r.field);
}
let value = cap.into_owned();
set_field_value(result, r.field, value);
@@ -154,7 +153,7 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt
for r in renames {
if matches!(r.field, ItemField::Group) {
let cap = r.pattern.replace_all(&grp.title, &r.new_name);
debug_if_enabled!("Renamed group {} to {} for {}", &grp.title, cap, target.name);
trace_if_enabled!("Renamed group {} to {cap} for {}", &grp.title, target.name);
grp.title = cap.into_owned();
}
}
@@ -305,7 +304,7 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo
(!user_targets.enabled && target.enabled) || (user_targets.enabled && user_targets.has_target(target.id))
}
async fn playlist_download_from_input(client: &Arc<reqwest::Client>, config: &Arc<Config>, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
async fn playlist_download_from_input(client: &reqwest::Client, config: &Arc<Config>, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let working_dir = &config.working_dir;
match input.input_type {
InputType::M3u => m3u::get_m3u_playlist(client, config, input, working_dir).await,
@@ -314,7 +313,7 @@ async fn playlist_download_from_input(client: &Arc<reqwest::Client>, config: &Ar
}
}
async fn process_source(client: Arc<reqwest::Client>, cfg: Arc<AppConfig>, source_idx: usize,
async fn process_source(client: &reqwest::Client, cfg: Arc<AppConfig>, source_idx: usize,
user_targets: Arc<ProcessTargets>, event_manager: Option<Arc<EventManager>>,
playlist_state: Option<&Arc<PlaylistStorageState>>,
) -> (Vec<InputStats>, Vec<TargetStats>, Vec<TuliproxError>) {
@@ -332,9 +331,9 @@ async fn process_source(client: Arc<reqwest::Client>, cfg: Arc<AppConfig>, sourc
let working_dir = &config.working_dir;
source_downloaded = true;
let start_time = Instant::now();
let (mut playlistgroups, mut error_list) = playlist_download_from_input(&client, &config, input).await;
let (mut playlistgroups, mut error_list) = playlist_download_from_input(client, &config, input).await;
let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
epg::get_xmltv(Arc::clone(&client), input, working_dir).await
epg::get_xmltv(client, input, working_dir).await
} else {
(None, vec![])
};
@@ -373,7 +372,7 @@ async fn process_source(client: Arc<reqwest::Client>, cfg: Arc<AppConfig>, sourc
for target in &source.targets {
let event_manager_clone = event_manager_clone.clone();
if is_target_enabled(target, &user_targets) {
match process_playlist_for_target(&cfg, Arc::clone(&client), &mut source_playlists, target, &mut input_stats, &mut errors, event_manager_clone, playlist_state).await {
match process_playlist_for_target(&cfg, client, &mut source_playlists, target, &mut input_stats, &mut errors, event_manager_clone, playlist_state).await {
Ok(()) => {
target_stats.push(TargetStats::success(&target.name));
}
@@ -407,7 +406,7 @@ fn create_input_stat(group_count: usize, channel_count: usize, error_count: usiz
}
}
async fn process_sources(client: Arc<reqwest::Client>, config: &Arc<AppConfig>, user_targets: Arc<ProcessTargets>,
async fn process_sources(client: &reqwest::Client, config: &Arc<AppConfig>, user_targets: Arc<ProcessTargets>,
event_manager: Option<Arc<EventManager>>, playlist_state: Option<&Arc<PlaylistStorageState>>,
) -> (Vec<SourceStats>, Vec<TuliproxError>) {
let mut async_tasks = JoinSet::new();
@@ -432,13 +431,13 @@ async fn process_sources(client: Arc<reqwest::Client>, config: &Arc<AppConfig>,
let usr_trgts = user_targets.clone();
let event_manager = event_manager.clone();
if process_parallel {
let http_client = Arc::clone(&client);
let http_client = client.clone();
let playlist_state = playlist_state.cloned();
async_tasks.spawn(async move {
// Hold the per-source lock for the full duration of this update.
let current_update_lock = update_lock;
let (input_stats, target_stats, mut res_errors) =
process_source(Arc::clone(&http_client), cfg, index, usr_trgts, event_manager, playlist_state.as_ref()).await;
process_source(&http_client, cfg, index, usr_trgts, event_manager, playlist_state.as_ref()).await;
shared_errors.lock().await.append(&mut res_errors);
if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) {
shared_stats.lock().await.push(process_stats);
@@ -447,7 +446,7 @@ async fn process_sources(client: Arc<reqwest::Client>, config: &Arc<AppConfig>,
});
} else {
let (input_stats, target_stats, mut res_errors) =
process_source(Arc::clone(&client), cfg, index, usr_trgts, event_manager, playlist_state).await;
process_source(client, cfg, index, usr_trgts, event_manager, playlist_state).await;
shared_errors.lock().await.append(&mut res_errors);
if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) {
shared_stats.lock().await.push(process_stats);
@@ -531,7 +530,7 @@ fn flatten_groups(playlistgroups: Vec<PlaylistGroup>) -> Vec<PlaylistGroup> {
#[allow(clippy::too_many_arguments)]
async fn process_playlist_for_target(app_config: &AppConfig,
client: Arc<reqwest::Client>,
client: &reqwest::Client,
playlists: &mut [FetchedPlaylist<'_>],
target: &ConfigTarget,
stats: &mut HashMap<String, InputStats>,
@@ -558,8 +557,8 @@ async fn process_playlist_for_target(app_config: &AppConfig,
let mut step = StepMeasure::new(&target.name, broadcast_step);
for provider_fpl in playlists.iter_mut() {
let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates);
playlist_resolve_series(app_config, Arc::clone(&client), target, errors, &pipe, provider_fpl, &mut processed_fpl).await;
playlist_resolve_vod(app_config, Arc::clone(&client), target, errors, &mut processed_fpl).await;
playlist_resolve_series(app_config, client, target, errors, &pipe, provider_fpl, &mut processed_fpl).await;
playlist_resolve_vod(app_config, client, target, errors, &mut processed_fpl).await;
// stats
let input_stats = stats.get_mut(&processed_fpl.input.name);
if let Some(stat) = input_stats {
@@ -580,7 +579,7 @@ async fn process_playlist_for_target(app_config: &AppConfig,
Ok(())
} else {
// Process Trakt categories
if trakt_playlist(&client, target, errors, &mut new_playlist).await {
if trakt_playlist(client, target, errors, &mut new_playlist).await {
step.tick("trakt categories");
}
@@ -596,7 +595,7 @@ async fn process_playlist_for_target(app_config: &AppConfig,
step.tick("assigning channel counter");
let config = app_config.config.load();
if process_watch(&config, &client, target, &flat_new_playlist).await {
if process_watch(&config, client, target, &flat_new_playlist).await {
step.tick("group watches");
}
let result = persist_playlist(app_config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, playlist_state).await;
@@ -605,8 +604,8 @@ async fn process_playlist_for_target(app_config: &AppConfig,
}
}
async fn trakt_playlist(client: &Arc<Client>, target: &ConfigTarget, errors: &mut Vec<TuliproxError>, playlist: &mut Vec<PlaylistGroup>) -> bool {
match process_trakt_categories_for_target(Arc::clone(client), playlist, target).await {
async fn trakt_playlist(client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec<TuliproxError>, playlist: &mut Vec<PlaylistGroup>) -> bool {
match process_trakt_categories_for_target(client, playlist, target).await {
Ok(Some(trakt_categories)) => {
if !trakt_categories.is_empty() {
info!("Adding {} Trakt categories to playlist", trakt_categories.len());
@@ -637,7 +636,7 @@ async fn process_epg(processed_fetched_playlists: &mut Vec<FetchedPlaylist<'_>>)
(new_epg, new_playlist)
}
async fn process_watch(cfg: &Config, client: &Arc<reqwest::Client>, target: &ConfigTarget, new_playlist: &[PlaylistGroup]) -> bool {
async fn process_watch(cfg: &Config, client: &reqwest::Client, target: &ConfigTarget, new_playlist: &[PlaylistGroup]) -> bool {
if let Some(watches) = &target.watch {
if default_as_default().eq_ignore_ascii_case(&target.name) {
error!("can't watch a target with no unique name");
@@ -657,10 +656,10 @@ async fn process_watch(cfg: &Config, client: &Arc<reqwest::Client>, target: &Con
}
}
pub async fn exec_processing(client: Arc<reqwest::Client>, app_config: Arc<AppConfig>, targets: Arc<ProcessTargets>, event_manager: Option<Arc<EventManager>>, playlist_state: Option<Arc<PlaylistStorageState>>) {
pub async fn exec_processing(client: &reqwest::Client, app_config: Arc<AppConfig>, targets: Arc<ProcessTargets>, event_manager: Option<Arc<EventManager>>, playlist_state: Option<Arc<PlaylistStorageState>>) {
let start_time = Instant::now();
let event_manager_clone = event_manager.clone();
let (stats, errors) = process_sources(Arc::clone(&client), &app_config, targets.clone(), event_manager_clone, playlist_state.as_ref()).await;
let (stats, errors) = process_sources(client, &app_config, targets.clone(), event_manager_clone, playlist_state.as_ref()).await;
// log errors
for err in &errors {
error!("{}", err.message);
@@ -677,7 +676,7 @@ pub async fn exec_processing(client: Arc<reqwest::Client>, app_config: Arc<AppCo
// print stats
info!("{stats_msg}");
// send stats
send_message_json(&client, MsgKind::Stats, messaging, stats_msg.as_str()).await;
send_message_json(client, MsgKind::Stats, messaging, stats_msg.as_str()).await;
}
Err(err) => error!("Failed to serialize playlist stats {err}"),
}
@@ -692,7 +691,7 @@ pub async fn exec_processing(client: Arc<reqwest::Client>, app_config: Arc<AppCo
events.send_event(EventMessage::PlaylistUpdate(PlaylistUpdateState::Failure));
}
if let Ok(error_msg) = serde_json::to_string(&serde_json::Value::Object(serde_json::map::Map::from_iter([("errors".to_string(), serde_json::Value::String(message))]))) {
send_message_json(&client, MsgKind::Error, messaging, error_msg.as_str()).await;
send_message_json(client, MsgKind::Error, messaging, error_msg.as_str()).await;
}
} else if let Some(events) = event_manager {
events.send_event(EventMessage::PlaylistUpdate(PlaylistUpdateState::Success));
backend/src/processing/processor/trakt.rs
+3 -4
View File
@@ -8,7 +8,6 @@ use shared::utils::{get_u32_from_serde_value};
use shared::utils::{CONSTANTS};
use crate::utils::{trace_if_enabled, with};
use log::{debug, info, trace, warn};
use std::sync::Arc;
use strsim::normalized_levenshtein;
fn extract_quality(value: &str) -> Option<&str> {
@@ -243,8 +242,8 @@ pub struct TraktCategoriesProcessor {
}
impl TraktCategoriesProcessor {
pub fn new(http_client: Arc<reqwest::Client>, trakt_config: &TraktConfig) -> Self {
let client = TraktClient::new(http_client, trakt_config.api.clone());
pub fn new(http_client: &reqwest::Client, trakt_config: &TraktConfig) -> Self {
let client = TraktClient::new(http_client.clone(), trakt_config.api.clone());
Self { client }
}
@@ -292,7 +291,7 @@ impl TraktCategoriesProcessor {
}
}
pub async fn process_trakt_categories_for_target(
http_client: Arc<reqwest::Client>,
http_client: &reqwest::Client,
playlist: &[PlaylistGroup],
target: &ConfigTarget,
) -> Result<Option<Vec<PlaylistGroup>>, Vec<TuliproxError>> {
backend/src/processing/processor/xtream.rs
+1 -2
View File
@@ -9,7 +9,6 @@ use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs::File;
use std::path::PathBuf;
use std::sync::Arc;
use crate::repository::bplustree::BPlusTree;
use crate::repository::storage_const;
use crate::repository::xtream_repository::xtream_get_record_file_path;
@@ -17,7 +16,7 @@ use crate::utils;
use crate::utils::xtream;
use serde_json::{from_str, to_string, Value};
pub(in crate::processing) async fn playlist_resolve_download_playlist_item(client: Arc<reqwest::Client>, pli: &PlaylistItem, input: &ConfigInput, errors: &mut Vec<TuliproxError>, resolve_delay: u16, cluster: XtreamCluster) -> Option<String> {
pub(in crate::processing) async fn playlist_resolve_download_playlist_item(client: &reqwest::Client, pli: &PlaylistItem, input: &ConfigInput, errors: &mut Vec<TuliproxError>, resolve_delay: u16, cluster: XtreamCluster) -> Option<String> {
let mut result = None;
let provider_id = pli.get_provider_id()?;
if let Some(info_url) = xtream::get_xtream_player_api_info_url(input, cluster, provider_id) {
backend/src/processing/processor/xtream_series.rs
+71 -44
View File
@@ -14,13 +14,12 @@ use crate::processing::processor::{handle_error, handle_error_and_return, create
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::{BufWriter, Write};
use std::sync::Arc;
use std::time::Instant;
use log::{error, info, log_enabled, warn, Level};
use crate::model::{XtreamSeriesEpisode, XtreamSeriesInfoEpisode};
use crate::utils;
use crate::processing::processor::xtream::normalize_json_content;
use crate::utils::bincode_serialize;
use crate::utils::{bincode_serialize, IO_BUFFER_SIZE};
create_resolve_options_function_for_xtream_target!(series);
@@ -32,27 +31,28 @@ fn write_series_episode_record_to_wal_file(
writer: &mut BufWriter<&File>,
provider_id: u32,
episode: &XtreamSeriesInfoEpisode,
) -> std::io::Result<()> {
) -> std::io::Result<usize> {
let series_episode = XtreamSeriesEpisode::from(episode);
if let Ok(content_bytes) = bincode_serialize(&series_episode) {
writer.write_all(&provider_id.to_le_bytes())?;
if let Ok(len) = u32::try_from(content_bytes.len()) {
let content_len = content_bytes.len();
if let Ok(len) = u32::try_from(content_len) {
writer.write_all(&len.to_le_bytes())?;
writer.write_all(&content_bytes)?;
} else {
error!("Cant write to WAL file, content length exceeds u32");
return Ok(content_len + 4usize)
}
error!("Cant write to WAL file, content length exceeds u32");
}
Ok(())
Ok(0)
}
fn should_update_series_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap<u32, u64>) -> (bool, u32, u64) {
should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_SERIES_INFO_LAST_MODIFIED)
}
async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc<reqwest::Client>, errors: &mut Vec<TuliproxError>,
async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, errors: &mut Vec<TuliproxError>,
fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool {
let mut processed_info_ids = read_processed_series_info_ids(cfg, errors, fpl).await;
let mut processed_info_ids: HashMap<u32, u64> = read_processed_series_info_ids(cfg, errors, fpl).await;
let mut fetched_in_run: HashSet<u32> = HashSet::new();
// we cant write to the indexed-document directly because of the write lock and time-consuming operation.
// All readers would be waiting for the lock and the app would be unresponsive.
@@ -64,57 +64,70 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc<reqwest::Clie
let mut record_writer = utils::file_writer(&wal_record_file);
let mut content_updated = false;
// TODO merge both filters to one
let series_info_count = fpl.playlistgroups.iter()
.filter(|&plg| plg.xtream_cluster == XtreamCluster::Series)
.flat_map(|plg| &plg.channels)
.filter(|&pli| pli.header.item_type == PlaylistItemType::SeriesInfo).count();
let series_info_iter = fpl.playlistgroups.iter_mut()
.filter(|plg| plg.xtream_cluster == XtreamCluster::Series)
.flat_map(|plg| &mut plg.channels)
.filter(|pli| pli.header.item_type == PlaylistItemType::SeriesInfo);
info!("Found {series_info_count} series info to resolve");
let start_time = Instant::now();
let mut last_log_time = Instant::now();
let mut processed_series_info_count = 0;
let mut last_processed_series_info_count = 0;
for pli in series_info_iter {
let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids);
if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
if let Some(content) = playlist_resolve_download_playlist_item(Arc::clone(&client), pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await {
let normalized_content = normalize_json_content(content);
handle_error_and_return!(write_series_info_to_wal_file(provider_id, ts, &normalized_content, &mut content_writer, &mut record_writer),
|err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}"))));
processed_info_ids.insert(provider_id, ts);
content_updated = true;
}
let mut write_counter = 0usize;
for plg in &mut fpl.playlistgroups {
if plg.xtream_cluster != XtreamCluster::Series {
continue;
}
if log_enabled!(Level::Info) {
processed_series_info_count += 1;
let elapsed = start_time.elapsed().as_secs();
if elapsed > 0 && ((processed_series_info_count - last_processed_series_info_count) > 50) && elapsed.is_multiple_of(30) {
info!("resolved {processed_series_info_count}/{series_info_count} series info");
last_processed_series_info_count = processed_series_info_count;
for pli in &mut plg.channels {
if pli.header.item_type != PlaylistItemType::SeriesInfo {
continue;
}
let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids);
if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await {
let normalized_content = normalize_json_content(content);
let normalized_str = normalized_content.as_str();
handle_error_and_return!(write_series_info_to_wal_file(provider_id, ts, normalized_str, &mut content_writer, &mut record_writer),
|err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}"))));
processed_info_ids.insert(provider_id, ts);
content_updated = true;
write_counter += normalized_str.len();
// periodic flush to bound BufWriter memory
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
if let Err(err) = content_writer.flush() {
errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}")));
}
if let Err(err) = record_writer.flush() {
errors.push(notify_err!(format!("Failed periodic flush of wal record writer {err}")));
}
}
}
}
if log_enabled!(Level::Info) {
processed_series_info_count += 1;
if last_log_time.elapsed().as_secs() >= 30 {
info!("resolved {processed_series_info_count}/{series_info_count} series info");
last_log_time = Instant::now();
}
}
}
}
if last_processed_series_info_count != processed_series_info_count {
info!("resolved {processed_series_info_count}/{series_info_count} series info");
}
info!("resolved {processed_series_info_count}/{series_info_count} series info");
// content_wal contains the provider_id and series_info with episode listing
// record_wal contains provider_id and timestamp
if content_updated {
handle_error!(content_writer.flush(),
|err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}"))));
|err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}"))));
handle_error!(record_writer.flush(),
|err| errors.push(notify_err!(format!("Failed to resolve vod tmdb, could not write to wal file {err}"))));
|err| errors.push(notify_err!(format!("Failed to resolve series tmdb, could not write to wal file {err}"))));
handle_error!(content_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}"))));
handle_error!(record_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info record to wal file {err}"))));
drop(content_writer);
drop(wal_content_file);
drop(record_writer);
drop(wal_content_file);
drop(wal_record_file);
handle_error!(xtream_update_input_info_file(cfg, fpl.input, &wal_content_path, XtreamCluster::Series).await,
|err| errors.push(err));
@@ -143,6 +156,8 @@ async fn process_series_info(
return result;
};
let mut write_counter = 0usize;
let _file_lock = app_config.file_locks.read_lock(&info_path).await;
// Contains the Series Info with episode listing
@@ -182,8 +197,19 @@ async fn process_series_info(
Ok(Some(mut series)) => {
for (episode, pli_episode) in &mut series {
let Some(provider_id) = &pli_episode.header.get_provider_id() else { continue; };
handle_error!(write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, episode),
|err| errors.push(info_err!(format!("Failed to write to series episode wal file: {err}"))));
match write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, episode) {
Ok(written_bytes) => {
write_counter += written_bytes;
// periodic flush to bound BufWriter memory
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
if let Err(err) = wal_writer.flush() {
errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}")));
}
}
}
Err(err) => {errors.push(info_err!(format!("Failed to write to series episode wal file: {err}"))) }
}
}
group_series.extend(series.into_iter().map(|(_, pli)| pli));
}
@@ -206,8 +232,9 @@ async fn process_series_info(
}
}
handle_error!(wal_writer.flush(),
|err| errors.push(notify_err!(format!("Failed to resolve series episodes, could not write to wal file {err}"))));
handle_error!(wal_writer.flush(), |err| errors.push(notify_err!(format!("Failed to resolve series episodes, could not write to wal file {err}"))));
handle_error!(wal_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}"))));
drop(wal_writer);
drop(wal_file);
handle_error!(xtream_update_input_series_episodes_record_from_wal_file(app_config, input, &wal_path).await,
@@ -217,7 +244,7 @@ async fn process_series_info(
pub async fn playlist_resolve_series(cfg: &AppConfig,
client: Arc<reqwest::Client>,
client: &reqwest::Client,
target: &ConfigTarget,
errors: &mut Vec<TuliproxError>,
pipe: &ProcessingPipe,
backend/src/processing/processor/xtream_vod.rs
+56 -54
View File
@@ -8,15 +8,14 @@ use crate::repository::xtream_repository::{write_vod_info_to_wal_file, xtream_up
use shared::error::{notify_err};
use crate::processing::processor::{handle_error, handle_error_and_return, create_resolve_options_function_for_xtream_target};
use shared::utils::{get_u32_from_serde_value, get_u64_from_serde_value, get_string_from_serde_value};
use crate::repository::xtream_repository::xtream_get_input_info;
use serde_json::{from_str, Map, Value};
use std::collections::{HashMap, HashSet};
use std::io::{Write};
use std::sync::Arc;
use std::time::Instant;
use log::{info, log_enabled, Level};
use crate::utils;
use crate::processing::processor::xtream::normalize_json_content;
use crate::utils::IO_BUFFER_SIZE;
create_resolve_options_function_for_xtream_target!(vod);
@@ -65,10 +64,12 @@ fn should_update_vod_info(pli: &mut PlaylistItem, processed_provider_ids: &HashM
should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_VOD_INFO_ADDED)
}
pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc<reqwest::Client>, target: &ConfigTarget, errors: &mut Vec<TuliproxError>, fpl: &mut FetchedPlaylist<'_>) {
pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec<TuliproxError>, fpl: &mut FetchedPlaylist<'_>) {
let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl);
if !resolve_movies { return; }
// TODO read existing WAL File and import it to avoid duplicate requests
// we cant write to the indexed-document directly because of the write lock and time-consuming operation.
// All readers would be waiting for the lock and the app would be unresponsive.
// We collect the content into a wal file and write it once we collected everything.
@@ -76,7 +77,7 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc<reqwest::C
let Some((wal_content_file, wal_record_file, wal_content_path, wal_record_path)) = create_resolve_info_wal_files(&config, fpl.input, XtreamCluster::Video)
else { return; };
let mut processed_info_ids = read_processed_vod_info_ids(app_config, errors, fpl).await;
let mut processed_info_ids: HashMap<u32, u64> = read_processed_vod_info_ids(app_config, errors, fpl).await;
let mut fetched_in_run: HashSet<u32> = HashSet::new();
let mut content_writer = utils::file_writer(&wal_content_file);
let mut record_writer = utils::file_writer(&wal_record_file);
@@ -87,78 +88,79 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc<reqwest::C
.flat_map(|plg| &plg.channels)
.filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Video).count();
let vod_info_iter = fpl.playlistgroups.iter_mut()
.flat_map(|plg| plg.channels.iter_mut())
.filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video);
info!("Found {vod_info_count} vod info to resolve");
let start_time = Instant::now();
let mut last_log_time = Instant::now();
let mut processed_vod_info_count = 0;
let mut last_processed_vod_info_count = 0;
let mut write_counter = 0usize;
for pli in vod_info_iter {
let (should_update, provider_id, _ts) = should_update_vod_info(pli, &processed_info_ids);
if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
if let Some(content) = playlist_resolve_download_playlist_item(Arc::clone(&client), pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await {
let normalized_content = normalize_json_content(content);
if let Some((provider_id, info_record)) = extract_info_record_from_vod_info(&normalized_content) {
let ts = info_record.ts;
handle_error_and_return!(write_vod_info_to_wal_file(provider_id, &normalized_content, &info_record, &mut content_writer, &mut record_writer),
|err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}"))));
processed_info_ids.insert(provider_id, ts);
content_updated = true;
for plg in &mut fpl.playlistgroups {
for pli in &mut plg.channels {
if pli.header.xtream_cluster != XtreamCluster::Video {
continue;
}
let (should_update, provider_id, _ts) = should_update_vod_info(pli, &processed_info_ids);
if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await {
let normalized_content = normalize_json_content(content);
let normalized_str: &str = &normalized_content;
if let Some((provider_id, info_record)) = extract_info_record_from_vod_info(normalized_str) {
let ts = info_record.ts;
handle_error_and_return!(write_vod_info_to_wal_file(provider_id, normalized_str, &info_record, &mut content_writer, &mut record_writer),
|err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}"))));
processed_info_ids.insert(provider_id, ts);
content_updated = true;
write_counter += normalized_str.len();
// periodic flush to bound BufWriter memory
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
if let Err(err) = content_writer.flush() {
errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}")));
}
if let Err(err) = record_writer.flush() {
errors.push(notify_err!(format!("Failed periodic flush of wal record writer {err}")));
}
}
// Update in-memory playlist items with the newly fetched vod info.
// This makes the data available for subsequent processing steps like STRM export.
pli.header.additional_properties = from_str::<Map<String, Value>>(normalized_str).ok().and_then(|info_doc| {
info_doc.get("info").cloned().map(|info_content| {
let mut wrapped_info = Map::new();
wrapped_info.insert("info".to_string(), info_content);
Value::Object(wrapped_info)
})
});
}
}
}
if log_enabled!(Level::Info) {
processed_vod_info_count += 1;
if last_log_time.elapsed().as_secs() >= 30 {
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
last_log_time = Instant::now();
}
}
}
if log_enabled!(Level::Info) {
processed_vod_info_count += 1;
let elapsed = start_time.elapsed().as_secs();
if elapsed > 0 && ((processed_vod_info_count - last_processed_vod_info_count) > 50) && elapsed.is_multiple_of(30) {
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
last_processed_vod_info_count = processed_vod_info_count;
}
}
}
if last_processed_vod_info_count != processed_vod_info_count {
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
}
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
if content_updated {
// TODO better approach for transactional updates is multiplexed WAL file.
// final flush & sync with proper error handling
handle_error!(content_writer.flush(),
|err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}"))));
handle_error!(record_writer.flush(),
|err| errors.push(notify_err!(format!("Failed to resolve vod tmdb, could not write to wal file {err}"))));
handle_error!(content_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync vod info to wal file {err}"))));
handle_error!(record_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync vod info record to wal file {err}"))));
// drop writers and files to release handles
drop(content_writer);
drop(record_writer);
drop(wal_content_file);
drop(wal_record_file);
handle_error!(xtream_update_input_info_file(app_config, fpl.input, &wal_content_path, XtreamCluster::Video).await,
|err| errors.push(err));
handle_error!(xtream_update_input_vod_record_from_wal_file(app_config, fpl.input, &wal_record_path).await,
|err| errors.push(err));
}
// Update in-memory playlist items with the newly fetched vod info.
// This makes the data available for subsequent processing steps like STRM export.
let vod_info_iter = fpl.playlistgroups.iter_mut()
.flat_map(|plg| &mut plg.channels)
.filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video);
for pli in vod_info_iter {
if let Some(provider_id) = pli.header.get_provider_id() {
if let Some(content) = xtream_get_input_info(app_config, fpl.input, provider_id, XtreamCluster::Video).await {
// Add the "info" section to the playlist item additional properties.
pli.header.additional_properties = from_str::<Map<String, Value>>(&content).ok().and_then(|info_doc| {
info_doc.get("info").cloned().map(|info_content| {
let mut wrapped_info = Map::new();
wrapped_info.insert("info".to_string(), info_content);
Value::Object(wrapped_info)
})
});
}
}
}
}
backend/src/repository/bplustree.rs
+132 -109
View File
@@ -1,7 +1,7 @@
use crate::utils;
use log::error;
use ruzstd::decoding::StreamingDecoder;
use ruzstd::encoding::{compress_to_vec, CompressionLevel};
use ruzstd::encoding::CompressionLevel;
use serde::{Deserialize, Serialize};
use shared::error::{str_to_io_error, to_io_error};
use std::fs::File;
@@ -40,6 +40,7 @@ fn u32_from_bytes(bytes: &[u8]) -> io::Result<u32> {
}
#[inline]
fn get_entry_index_upper_bound<K>(keys: &[K], key: &K) -> usize
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
@@ -58,15 +59,14 @@ where
}
fn query_tree_le<K, V, R: Read + Seek>(file: &mut R, key: &K) -> Option<V>
fn query_tree_le<K, V, R: Read + Seek>(file: &mut R, buffer: &mut Vec<u8>, key: &K) -> Option<V>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
let mut offset = 0;
let mut buffer = vec![0u8; BLOCK_SIZE];
loop {
match BPlusTreeNode::<K, V>::deserialize_from_block(file, &mut buffer, offset, false) {
match BPlusTreeNode::<K, V>::deserialize_from_block(file, buffer, offset, false) {
Ok((node, pointers)) => {
if node.is_leaf {
let idx = get_entry_index_upper_bound::<K>(&node.keys, key);
@@ -272,131 +272,156 @@ where
self.children.iter().for_each(|child| child.traverse(visit));
}
fn serialize_to_block<W: Write + Seek>(&self, file: &mut W, buffer: &mut Vec<u8>, offset: u64) -> io::Result<u64> {
fn serialize_to_block<W: Write + Seek>(
&self,
file: &mut W,
buffer: &mut Vec<u8>, // must be length BLOCK_SIZE
offset: u64,
) -> io::Result<u64> {
// Keep backward-compatible on-disk layout with minimal allocations.
let mut current_offset = offset;
// zero the working block and set node type
buffer.fill(0_u8);
let buffer_slice = &mut buffer[..];
// Write node type (leaf or internal)
buffer_slice[0] = u8::from(self.is_leaf);
let mut write_pos = FLAG_SIZE;
// Serialize and write keys
// ---- Write keys (length + bytes) into the first block ----
let keys_encoded = bincode_serialize(&self.keys)?;
let keys_bytes_len = keys_encoded.len();
buffer_slice[write_pos..write_pos + LEN_SIZE].copy_from_slice(&(u32::try_from(keys_bytes_len).map_err(to_io_error)?).to_le_bytes());
let keys_len = keys_encoded.len();
buffer_slice[write_pos..write_pos + LEN_SIZE]
.copy_from_slice(&u32::try_from(keys_len).map_err(to_io_error)?.to_le_bytes());
write_pos += LEN_SIZE;
buffer_slice[write_pos..write_pos + keys_bytes_len].copy_from_slice(&keys_encoded);
write_pos += keys_bytes_len;
// NOTE: By design of the legacy layout, keys are expected to fit into the first block.
buffer_slice[write_pos..write_pos + keys_len].copy_from_slice(&keys_encoded);
write_pos += keys_len;
drop(keys_encoded);
let mut remaining: Option<Vec<u8>> = None;
// If leaf, serialize and write values
// Prepare pointer offset for internal nodes (must be within first block)
let pointer_offset_within_first_block = if self.is_leaf { 0u64 } else { offset + write_pos as u64 };
// ---- Leaf values (optional) ----
if self.is_leaf {
// Encode values and decide compression exactly like the old layout
let values_encoded = bincode_serialize(&self.values)?;
let use_compression = values_encoded.len() + write_pos >= BLOCK_SIZE;
let compression_byte = if use_compression { 1u8.to_le_bytes() } else { 0u8.to_le_bytes() };
buffer_slice[write_pos..=write_pos].copy_from_slice(&compression_byte);
// Compression flag
buffer_slice[write_pos] = u8::from(use_compression);
write_pos += FLAG_SIZE;
// Content bytes (possibly compressed)
let content_bytes = if use_compression {
{
// use ruzstd::io::Read;
compress_to_vec(&*values_encoded, CompressionLevel::Fastest)
}
// ruzstd expects a Read implementor; wrap the slice in a Cursor
ruzstd::encoding::compress_to_vec(std::io::Cursor::new(values_encoded.as_slice()), CompressionLevel::Fastest)
} else {
values_encoded
};
let values_bytes_len = content_bytes.len();
buffer_slice[write_pos..write_pos + LEN_SIZE].copy_from_slice(&(u32::try_from(values_bytes_len).map_err(to_io_error)?).to_le_bytes());
// Write content length
let content_len = content_bytes.len();
buffer_slice[write_pos..write_pos + LEN_SIZE]
.copy_from_slice(&u32::try_from(content_len).map_err(to_io_error)?.to_le_bytes());
write_pos += LEN_SIZE;
let mut value_bytes_to_write = values_bytes_len;
let bytes_to_write = write_pos + values_bytes_len;
if bytes_to_write > BLOCK_SIZE {
value_bytes_to_write = BLOCK_SIZE - write_pos;
remaining = Some(content_bytes[value_bytes_to_write..].to_vec());
// Copy as many content bytes as fit into the first block
let space_left = BLOCK_SIZE.saturating_sub(write_pos);
let first_copy = std::cmp::min(space_left, content_len);
buffer_slice[write_pos..write_pos + first_copy]
.copy_from_slice(&content_bytes[..first_copy]);
// Write the full first block
file.seek(SeekFrom::Start(offset))?;
file.write_all(&buffer_slice[..BLOCK_SIZE])?;
current_offset += BLOCK_SIZE as u64;
// Stream remaining content (if any) block-by-block without extra allocations
let mut pos = first_copy;
while pos < content_len {
let remaining = content_len - pos;
let chunk = std::cmp::min(remaining, BLOCK_SIZE);
// Copy the next chunk
buffer_slice[..chunk].copy_from_slice(&content_bytes[pos..pos + chunk]);
// If this is a partial block, zero the tail only once
if chunk < BLOCK_SIZE {
buffer_slice[chunk..BLOCK_SIZE].fill(0u8);
}
file.write_all(&buffer_slice[..BLOCK_SIZE])?; // always write full block (pad with zeros)
current_offset += BLOCK_SIZE as u64;
pos += chunk;
}
buffer_slice[write_pos..write_pos + value_bytes_to_write].copy_from_slice(&content_bytes[..value_bytes_to_write]);
// write_pos += value_bytes_to_write;
// Free content buffer ASAP
drop(content_bytes);
} else {
// Internal node: write out the full first block now
file.seek(SeekFrom::Start(offset))?;
file.write_all(&buffer_slice[..BLOCK_SIZE])?;
current_offset += BLOCK_SIZE as u64;
}
// Write the complete buffer to file, do not optimize to real filled size,
file.seek(SeekFrom::Start(offset))?;
file.write_all(&buffer_slice[..BLOCK_SIZE])?; // use BLOCK_SIZE
current_offset += BLOCK_SIZE as u64;
if let Some(mut left_over) = remaining {
// we calculate the needed blocks
let left_over_len = left_over.len();
if !left_over_len.is_multiple_of(BLOCK_SIZE) {
let padding = BLOCK_SIZE - (left_over_len % BLOCK_SIZE);
left_over.extend(vec![0u8; padding]);
}
let total_len = left_over.len();
file.write_all(&left_over[..total_len])?;
current_offset += total_len as u64;
}
// we need to write the pointers after the children are persisted,
// because we need the offsets, which we get after persisting.
// ---- Internal node pointers: serialize children, then write pointer list into first block ----
if !self.is_leaf {
let pointer_offset = offset + write_pos as u64;
let mut pointer = Vec::with_capacity(self.children.len());
let pointer_offset = pointer_offset_within_first_block; // inside the node's first block
let mut pointer_vec: Vec<u64> = Vec::with_capacity(self.children.len());
for child in &self.children {
pointer.push(current_offset);
pointer_vec.push(current_offset);
current_offset = child.serialize_to_block(file, buffer, current_offset)?;
}
let pointer_encoded = bincode_serialize(&pointer)?;
let pointer_bytes_len = u32::try_from(pointer_encoded.len()).map_err(to_io_error)?;
let pointer_encoded = bincode_serialize(&pointer_vec)?;
let pointer_len = u32::try_from(pointer_encoded.len()).map_err(to_io_error)?;
file.seek(SeekFrom::Start(pointer_offset))?;
file.write_all(&pointer_bytes_len.to_le_bytes())?;
file.write_all(&pointer_len.to_le_bytes())?;
file.write_all(&pointer_encoded)?;
}
Ok(current_offset)
}
fn deserialize_from_block<R: Read + Seek>(file: &mut R, buffer: &mut Vec<u8>, offset: u64, nested: bool) -> io::Result<(Self, Option<Vec<u64>>)> {
fn deserialize_from_block<R: Read + Seek>(
file: &mut R,
buffer: &mut Vec<u8>,
offset: u64,
nested: bool,
) -> io::Result<(Self, Option<Vec<u64>>)> {
// Read the full first block into buffer (always aligned on-disk)
file.seek(SeekFrom::Start(offset))?;
file.read_exact(buffer)?;
// Read the node type directly from buffer
// Node type
let is_leaf = buffer[0] == 1u8;
let mut read_pos = FLAG_SIZE;
// Deserialize keys
// ---- Keys ----
let keys_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize;
read_pos += LEN_SIZE;
let keys: Vec<K> = bincode_deserialize(&buffer[read_pos..read_pos + keys_length])?;
read_pos += keys_length;
// Deserialize values if leaf node
let values = if is_leaf {
let use_compression = u8::from_le_bytes(buffer[read_pos..=read_pos].try_into().unwrap_or([0u8])) == 1;
// ---- Values for leaf nodes ----
let values: Vec<V> = if is_leaf {
// compression flag
let use_compression = buffer[read_pos] == 1u8;
read_pos += FLAG_SIZE;
// content length
let values_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize;
read_pos += LEN_SIZE;
// Fast path: everything is inside the first block
let bytes_available_on_block = BLOCK_SIZE - read_pos;
let content_bytes = if values_length > bytes_available_on_block {
let mut left_over_bytes = values_length;
let mut content_chunk = Vec::from(&buffer[read_pos..read_pos + bytes_available_on_block]);
left_over_bytes -= bytes_available_on_block;
while left_over_bytes > 0 {
file.read_exact(buffer)?;
let bytes_to_read = if left_over_bytes >= BLOCK_SIZE {
BLOCK_SIZE
} else {
left_over_bytes
};
content_chunk.extend(&buffer[..bytes_to_read]);
left_over_bytes -= bytes_to_read;
}
content_chunk[..values_length].to_vec()
let mut content_bytes: Vec<u8> = vec![0u8; values_length];
if values_length <= bytes_available_on_block {
content_bytes[..values_length]
.copy_from_slice(&buffer[read_pos..read_pos + values_length]);
} else {
buffer[read_pos..read_pos + values_length].to_vec()
};
// Copy what's left in the first block
content_bytes[..bytes_available_on_block]
.copy_from_slice(&buffer[read_pos..read_pos + bytes_available_on_block]);
// Read the remaining bytes directly into the final buffer
file.read_exact(&mut content_bytes[bytes_available_on_block..])?;
}
let values_bytes = if use_compression {
decode_content(&content_bytes).unwrap_or(content_bytes)
@@ -404,32 +429,27 @@ where
content_bytes
};
let values: Vec<V> = bincode_deserialize(&values_bytes)?;
read_pos += values_length;
values
bincode_deserialize(&values_bytes)?
} else {
vec![]
Vec::new()
};
// Deserialize children indices if internal node
let (children, children_pointer) = if is_leaf {
(vec![], None)
// ---- Pointers for internal nodes ----
let (children, children_pointer): (Vec<Self>, Option<Vec<u64>>) = if is_leaf {
(Vec::new(), None)
} else {
let pointers_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize;
read_pos += LEN_SIZE;
let pointers: Vec<u64> = bincode_deserialize(&buffer[read_pos..read_pos + pointers_length])?;
if nested {
let nodes: Result<Vec<Self>, io::Error> = pointers
.iter()
.map(|pointer| {
Self::deserialize_from_block(file, buffer, *pointer, nested)
.map(|(node, _)| node)
})
.collect();
(nodes?, None)
let mut nodes = Vec::with_capacity(pointers.len());
for &ptr in &pointers {
let (child, _) = Self::deserialize_from_block(file, buffer, ptr, nested)?;
nodes.push(child);
}
(nodes, None)
} else {
(vec![], Some(pointers))
(Vec::new(), Some(pointers))
}
};
@@ -437,8 +457,8 @@ where
}
}
fn decode_content(content_bytes: &Vec<u8>) -> Option<Vec<u8>> {
if let Ok(mut decoder) = StreamingDecoder::new(&**content_bytes) {
fn decode_content(content_bytes: &[u8]) -> Option<Vec<u8>> {
if let Ok(mut decoder) = StreamingDecoder::new(content_bytes) {
let mut result = Vec::with_capacity(content_bytes.len());
if decoder.read_to_end(&mut result).is_ok() {
return Some(result);
@@ -592,15 +612,14 @@ where
}
}
fn query_tree<K, V, R: Read + Seek>(file: &mut R, key: &K) -> Option<V>
fn query_tree<K, V, R: Read + Seek>(file: &mut R, buffer: &mut Vec<u8>, key: &K) -> Option<V>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
let mut offset = 0;
let mut buffer = vec![0u8; BLOCK_SIZE];
loop {
match BPlusTreeNode::<K, V>::deserialize_from_block(file, &mut buffer, offset, false) {
match BPlusTreeNode::<K, V>::deserialize_from_block(file, buffer, offset, false) {
Ok((node, pointers)) => {
if node.is_leaf {
return match node.keys.binary_search(key) {
@@ -655,6 +674,7 @@ where
///
pub struct BPlusTreeQuery<K, V> {
file: BufReader<File>,
buffer: Vec<u8>,
_marker_k: PhantomData<K>,
_marker_v: PhantomData<V>,
}
@@ -668,6 +688,7 @@ where
let file = is_file_valid(file)?;
Ok(Self {
file: utils::file_reader(file),
buffer: vec![0u8; BLOCK_SIZE],
_marker_k: PhantomData,
_marker_v: PhantomData,
})
@@ -679,7 +700,7 @@ where
}
pub fn query(&mut self, key: &K) -> Option<V> {
query_tree(&mut self.file, key)
query_tree(&mut self.file, &mut self.buffer, key)
}
/// On-disk: find largest key <= `key` and return owned V (cloned/deserialized)
@@ -692,7 +713,7 @@ where
// if seek fails, still try to query — but bail out with None
return None;
}
query_tree_le(file, key)
query_tree_le(file, &mut self.buffer, key)
}
// pub fn traverse<F>(&mut self, mut visit: F)
@@ -705,6 +726,8 @@ where
pub struct BPlusTreeUpdate<K, V> {
file: File,
read_buffer: Vec<u8>,
write_buffer: Vec<u8>,
_marker_k: PhantomData<K>,
_marker_v: PhantomData<V>,
}
@@ -721,6 +744,8 @@ where
let file = is_file_valid(utils::open_read_write_file(filepath)?)?;
Ok(Self {
file,
read_buffer: vec![0u8; BLOCK_SIZE],
write_buffer: vec![0u8; BLOCK_SIZE],
_marker_k: PhantomData,
_marker_v: PhantomData,
})
@@ -728,22 +753,20 @@ where
pub fn query(&mut self, key: &K) -> Option<V> {
let mut reader = utils::file_reader(&mut self.file);
query_tree(&mut reader, key)
query_tree(&mut reader, &mut self.read_buffer, key)
}
fn serialize_node(&mut self, offset: u64, node: &BPlusTreeNode<K, V>) -> io::Result<u64> {
let mut buffer = vec![0u8; BLOCK_SIZE];
let result = node.serialize_to_block(&mut self.file, &mut buffer, offset);
let result = node.serialize_to_block(&mut self.file, &mut self.write_buffer, offset);
self.file.flush()?;
result
}
pub fn update(&mut self, key: &K, value: V) -> io::Result<u64> {
let mut offset = 0;
let mut buffer = vec![0u8; BLOCK_SIZE];
let mut reader = utils::file_reader(&mut self.file);
loop {
match BPlusTreeNode::<K, V>::deserialize_from_block(&mut reader, &mut buffer, offset, false) {
match BPlusTreeNode::<K, V>::deserialize_from_block(&mut reader, &mut self.read_buffer, offset, false) {
Ok((mut node, pointers)) => {
if node.is_leaf {
return match node.keys.binary_search(key) {
@@ -776,7 +799,7 @@ where
/// On-disk update helper: find largest key <= `key`.
pub fn query_le(&mut self, key: &K) -> Option<V> {
let mut reader = utils::file_reader(&mut self.file);
query_tree_le(&mut reader, key)
query_tree_le(&mut reader, &mut self.read_buffer, key)
}
}
backend/src/repository/epg_repository.rs
+32 -18
View File
@@ -2,42 +2,56 @@ use crate::model::Epg;
use crate::model::{Config, ConfigTarget, TargetOutput};
use crate::repository::m3u_repository::m3u_get_epg_file_path;
use crate::repository::xtream_repository::{xtream_get_epg_file_path, xtream_get_storage_path};
use crate::utils::debug_if_enabled;
use crate::utils::{async_file_writer, debug_if_enabled};
use shared::error::{notify_err, TuliproxError};
use std::path::Path;
use tokio::io::AsyncWriteExt;
// Due to an error in quick_xml we cant write doc type through event. The quotes are escaped and the xml file is invalid.
const XML_PREAMBLE: &str = r#"<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE tv SYSTEM "xmltv.dtd">
"#;
// Due to a bug in quick_xml we cannot write the DOCTYPE via event; quotes are escaped and the XML becomes invalid.
// Keep the manual header/doctype write workaround below.
//
// // XML Header
// // XML Header via events (DO NOT USE, kept for documentation):
// writer.write_event_async(quick_xml::events::Event::Decl(quick_xml::events::BytesDecl::new("1.0", Some("utf-8"), None)))
// .await.map_err(|e| notify_err!(format!("failed to write XML header: {}", e)))?;
//
// // DOCTYPE
// // DOCTYPE via events (DO NOT USE):
// writer.write_event_async(quick_xml::events::Event::DocType(quick_xml::events::BytesText::new(r#"tv SYSTEM "xmltv.dtd""#)))
// .await.map_err(|e| notify_err!(format!("failed to write doctype: {}", e)))?;
pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path) -> Result<(), TuliproxError> {
let file = tokio::fs::File::create(path).await
.map_err(|e| notify_err!(format!("failed to create epg file: {}", e)))?;
let mut buf_writer = tokio::io::BufWriter::new(file);
.map_err(|e| notify_err!(format!("failed to create epg file {}: {}", path.display(), e)))?;
// Work-Around BytesText DocType escape, see below
buf_writer.write_all(b"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n").await
.map_err(|e| notify_err!(format!("failed to write XML header: {}", e)))?;
// Use a larger buffer for sequential writes to reduce syscalls
let mut buf_writer = async_file_writer(file);
buf_writer.write_all(b"<!DOCTYPE tv SYSTEM \"xmltv.dtd\">\n").await
.map_err(|e| notify_err!(format!("failed to write doctype: {}", e)))?;
// Header/DOCTYPE workaround: write both lines in a single call for fewer syscalls
buf_writer
.write_all(XML_PREAMBLE.as_bytes())
.await
.map_err(|e| notify_err!(format!("failed to write XML preamble {}: {}", path.display(), e)))?;
// EPG content streamed via quick_xml writer (compact output)
let mut writer = quick_xml::writer::Writer::new(buf_writer);
epg
.write_to_async(&mut writer)
.await
.map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?;
// EPG Content
epg.write_to_async(&mut writer).await.map_err(|e| notify_err!(format!("failed to write epg: {}", e)))?;
// Ensure buffers are flushed to the OS and capture any I/O error
let mut buf_writer = writer.into_inner();
buf_writer
.flush()
.await
.map_err(|e| notify_err!(format!("failed to flush epg {}: {}", path.display(), e)))?;
let inner = writer.get_mut(); // Zugriff auf den BufWriter<tokio::fs::File>
inner.flush().await.map_err(|e| notify_err!(format!("failed to flush epg: {}", e)))?;
buf_writer.shutdown().await.map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?;
debug_if_enabled!("Epg for target {} written to {}", target.name, path.to_str().unwrap_or("?"));
debug_if_enabled!("Epg for target {} written to {}", target.name, path.display());
Ok(())
}
@@ -48,7 +62,7 @@ pub async fn epg_write(cfg: &Config, target: &ConfigTarget, target_path: &Path,
match xtream_get_storage_path(cfg, &target.name) {
Some(path) => {
let epg_path = xtream_get_epg_file_path(&path);
debug_if_enabled!("writing xtream epg to {}", epg_path.to_str().unwrap_or("?"));
debug_if_enabled!("writing xtream epg to {}", epg_path.display());
epg_write_file(target, epg_data, &epg_path).await?;
}
None => return Err(notify_err!(format!("failed to serialize epg for target: {}, storage path not found", target.name))),
@@ -56,7 +70,7 @@ pub async fn epg_write(cfg: &Config, target: &ConfigTarget, target_path: &Path,
}
TargetOutput::M3u(_) => {
let path = m3u_get_epg_file_path(target_path);
debug_if_enabled!("writing m3u epg to {}", path.to_str().unwrap_or("?"));
debug_if_enabled!("writing m3u epg to {}", path.display());
epg_write_file(target, epg_data, &path).await?;
}
TargetOutput::Strm(_) | TargetOutput::HdHomeRun(_) => {}
backend/src/repository/indexed_document.rs
+68 -42
View File
@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile;
use shared::error::{str_to_io_error, to_io_error};
use crate::utils;
use crate::utils::{bincode_deserialize, bincode_serialize};
use crate::utils::{bincode_deserialize, bincode_serialize, IO_BUFFER_SIZE};
const BLOCK_SIZE: usize = 4096;
const LEN_SIZE: usize = 4;
@@ -163,54 +163,67 @@ where
where
T: ?Sized + serde::Serialize,
{
let encoded_bytes = bincode_serialize(doc).map_err(|_| Error::new(ErrorKind::InvalidData, "Failed to serialize document"))?;
let mut new_record_appended = false; // do i need to change the index and set the new offset
// Determine target position (overwrite in place or append)
if let Some(&offset) = self.index_tree.query(&doc_id) {
// Existing entry: check size and choose strategy
self.main_file.seek(SeekFrom::Start(u64::from(offset)))?;
let size = IndexedDocument::read_content_size(&mut self.main_file)?;
if size == encoded_bytes.len() {
// check if it is equal
let mut record_buffer = vec![0; size];
// record_buffer.resize(size, 0);
let old_size = IndexedDocument::read_content_size(&mut self.main_file)?;
self.main_file.read_exact(&mut record_buffer)?;
if record_buffer == encoded_bytes {
return Ok(());
}
}
// Prepare new payload (bincode utils, compatible with project)
let encoded = bincode_serialize(doc)
.map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?;
let new_size: SizeType = SizeType::try_from(encoded.len() as u64)
.map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?;
if encoded_bytes.len() > size {
// does not fit we need to append, file is fragmented
if usize::try_from(new_size).unwrap_or(usize::MAX) > old_size {
// Does not fit -> append at end of file (mark fragmentation)
if !self.fragmented {
self.fragmented = true;
IndexedDocument::write_fragmentation(&mut self.main_file, true)?;
}
self.main_file.seek(SeekFrom::End(0))?;
new_record_appended = true;
} else {
self.main_file.seek(SeekFrom::Start(u64::from(offset)))?;
}
} else {
self.main_file.seek(SeekFrom::End(0))?;
new_record_appended = true;
}
// We write: length + payload
self.dirty = true;
self.main_file.write_all(&new_size.to_le_bytes())?;
self.main_file.write_all(&encoded)?;
self.index_tree.insert(doc_id, self.main_offset);
// Increase main_offset: LEN_SIZE + new_size
let written_bytes_u64 = (LEN_SIZE as u64) + u64::from(new_size);
let written_bytes: SizeType = SizeType::try_from(written_bytes_u64)
.map_err(|e| str_to_io_error(&format!("Written byte count overflow for {}: {e}", self.main_path.display())))?;
self.main_offset = self.main_offset.checked_add(written_bytes)
.ok_or_else(|| str_to_io_error(&format!("main_offset overflow while appending to {}", self.main_path.display())))?;
return Ok(());
}
// Fits (<=): overwrite at the same position
self.main_file.seek(SeekFrom::Start(u64::from(offset)))?;
self.dirty = true;
self.main_file.write_all(&new_size.to_le_bytes())?;
self.main_file.write_all(&encoded)?;
return Ok(());
}
// New entry -> append
self.main_file.seek(SeekFrom::End(0))?;
// Determine size and write it, then write payload
let encoded = bincode_serialize(doc)
.map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?;
let new_size: SizeType = SizeType::try_from(encoded.len() as u64)
.map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?;
self.dirty = true;
self.main_file
.write_all(&new_size.to_le_bytes())
.map_err(|e| str_to_io_error(&format!("Failed to write length prefix to {}: {e}", self.main_path.display())))?;
self.main_file
.write_all(&encoded)
.map_err(|e| str_to_io_error(&format!("Failed to write document payload to {}: {e}", self.main_path.display())))?;
let encoded_bytes_len = SizeType::try_from(encoded_bytes.len()).map_err(to_io_error)?;
self.main_file.write_all(&encoded_bytes_len.to_le_bytes())?;
match utils::check_write(&self.main_file.write_all(&encoded_bytes)) {
Ok(()) => {
if new_record_appended {
self.index_tree.insert(doc_id, self.main_offset);
let written_bytes = SizeType::try_from(encoded_bytes.len() + LEN_SIZE).map_err(to_io_error)?;
self.main_offset += written_bytes;
}
}
Err(err) => {
return Err(str_to_io_error(&format!("failed to write document: {} - {}", self.main_path.display(), err)));
}
}
self.index_tree.insert(doc_id, self.main_offset);
let written_bytes_u64 = (LEN_SIZE as u64) + u64::from(new_size);
let written_bytes: SizeType = SizeType::try_from(written_bytes_u64)
.map_err(|e| str_to_io_error(&format!("Written byte count overflow for {}: {e}", self.main_path.display())))?;
self.main_offset = self.main_offset.checked_add(written_bytes)
.ok_or_else(|| str_to_io_error(&format!("main_offset overflow while appending to {}", self.main_path.display())))?;
Ok(())
}
}
@@ -237,6 +250,7 @@ where
{
main_file: BufReader<File>,
index_tree: IndexedDocumentIndex<K>,
buffer: Vec<u8>,
t_type: PhantomData<T>,
}
@@ -252,8 +266,10 @@ where
let index_tree = IndexedDocumentIndex::<K>::load(index_path)?;
Ok(Self {
main_file: utils::file_reader(main_file),
// Larger read buffer for sequential access
main_file: BufReader::with_capacity(BLOCK_SIZE * 64, main_file),
index_tree,
buffer: Vec::with_capacity(BLOCK_SIZE),
t_type: PhantomData,
})
} else {
@@ -264,9 +280,9 @@ where
if let Some(offset) = self.index_tree.query(doc_id) {
self.main_file.seek(SeekFrom::Start(u64::from(*offset)))?;
let buf_size = IndexedDocument::read_content_size(&mut self.main_file)?;
let mut buffer: Vec<u8> = vec![0; buf_size];
self.main_file.read_exact(&mut buffer)?;
if let Ok(item) = bincode_deserialize::<T>(&buffer) {
self.buffer.resize(buf_size, 0u8);
self.main_file.read_exact(&mut self.buffer[..buf_size])?;
if let Ok(item) = bincode_deserialize::<T>(&self.buffer[..buf_size]) {
return Ok(item);
}
}
@@ -308,7 +324,8 @@ where
Ok(file) => {
Ok(Self {
main_path: main_path.to_path_buf(),
main_file: utils::file_reader(file),
// Larger read buffer for sequential iteration
main_file: BufReader::with_capacity(BLOCK_SIZE * 128, file),
offsets,
index: 0,
failed: false,
@@ -467,6 +484,8 @@ where
self.index_tree.traverse(|keys, values| {
keys.iter().zip(values.iter()).for_each(|(key, &offset)| key_offset.push((key.clone(), offset)));
});
// For better disk locality: sort by offset (sequential reads)
key_offset.sort_by_key(|(_, off)| *off);
let mut gc_writer = utils::file_writer(&gc_file);
let fragmented_byte = 0u8.to_le_bytes();
@@ -475,6 +494,8 @@ where
let mut gc_offset = 1usize; // offset is 1 because of fragment bit
let mut buffer: Vec<u8> = Vec::with_capacity(BLOCK_SIZE);
let mut size_bytes = [0u8; LEN_SIZE];
let mut write_counter = 0usize;
for (key, offset) in key_offset {
// read old content
self.main_file.seek(SeekFrom::Start(u64::from(offset)))?;
@@ -489,6 +510,11 @@ where
gc_writer.write_all(&size_bytes)?;
gc_writer.write_all(&buffer[0..buf_size])?;
write_counter += buf_size;
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
gc_writer.flush()?;
}
let pointer = OffsetPointer::try_from(gc_offset).map_err(to_io_error)?;
self.index_tree.insert(key, pointer);
backend/src/repository/m3u_playlist_iterator.rs
+52 -27
View File
@@ -10,6 +10,7 @@ use crate::repository::storage_const;
use crate::repository::user_repository::user_get_bouquet_filter;
use crate::utils::FileReadGuard;
use std::collections::HashSet;
// concat_string! macro from shared utils is used for efficient String building
#[allow(clippy::struct_excessive_bools)]
pub struct M3uPlaylistIterator {
@@ -67,26 +68,39 @@ impl M3uPlaylistIterator {
}
fn get_rewritten_url(&self, m3u_pli: &M3uPlaylistItem, typed: bool, prefix_path: &str) -> String {
if typed {
let stream_type = match m3u_pli.item_type {
// Build URL efficiently with a single allocation using concat_string! macro
let stream_type: &str = if typed {
match m3u_pli.item_type {
PlaylistItemType::Live
| PlaylistItemType::Catchup
| PlaylistItemType::LiveUnknown
| PlaylistItemType::LiveHls
| PlaylistItemType::LiveDash => "live",
PlaylistItemType::Video => "movie",
PlaylistItemType::Series
| PlaylistItemType::SeriesInfo => "series",
};
format!("{}/{prefix_path}/{stream_type}/{}/{}/{}",
&self.base_url,
&self.username,
&self.password,
m3u_pli.virtual_id
PlaylistItemType::Series | PlaylistItemType::SeriesInfo => "series",
}
} else {
""
};
let mut cap = self.base_url.len()
+ prefix_path.len()
+ self.username.len()
+ self.password.len()
+ 32; // separators and id
if typed { cap += stream_type.len() + 1; }
if typed {
shared::concat_string!(
cap = cap;
&self.base_url, "/", prefix_path, "/", stream_type, "/",
&self.username, "/", &self.password, "/", m3u_pli.virtual_id
)
} else {
format!("{}/{prefix_path}/{}/{}/{}",
&self.base_url, &self.username, &self.password, m3u_pli.virtual_id
shared::concat_string!(
cap = cap;
&self.base_url, "/", prefix_path, "/",
&self.username, "/", &self.password, "/", m3u_pli.virtual_id
)
}
}
@@ -101,14 +115,15 @@ impl M3uPlaylistIterator {
fn get_next(&mut self) -> Option<(M3uPlaylistItem, bool)> {
let entry = if let Some(set) = &self.filter {
if let Some((current_item, _)) = self.lookup_item.take() {
let next_valid = self.reader.find(|(pli, _)| set.contains(&pli.group.clone()));
// Avoid cloning strings while filtering
let next_valid = self.reader.find(|(pli, _)| set.contains(pli.group.as_str()));
self.lookup_item = next_valid;
let has_next = self.lookup_item.is_some();
Some((current_item, has_next))
} else {
let current_item = self.reader.find(|(item, _)| set.contains(&item.group.clone()));
let current_item = self.reader.find(|(item, _)| set.contains(item.group.as_str()));
if let Some((item, _)) = current_item {
self.lookup_item = self.reader.find(|(item, _)| set.contains(&item.group.clone()));
self.lookup_item = self.reader.find(|(item, _)| set.contains(item.group.as_str()));
let has_next = self.lookup_item.is_some();
Some((item, has_next))
} else {
@@ -121,19 +136,29 @@ impl M3uPlaylistIterator {
// TODO hls and unknown reverse proxy
entry.map(|(mut m3u_pli, has_next)| {
let is_redirect = self.proxy_type.is_redirect(m3u_pli.item_type) || self.target_options.as_ref().and_then(|o| o.force_redirect.as_ref()).is_some_and(|f| f.has_cluster(m3u_pli.item_type));
let should_rewrite_urls = if is_redirect { self.mask_redirect_url} else { true };
let rewrite_urls = if should_rewrite_urls {
Some((self.get_stream_url(&m3u_pli, self.include_type_in_url), if self.rewrite_resource { Some(self.get_resource_url(&m3u_pli)) } else { None }))
} else {
None
};
let url = m3u_pli.url.clone();
let (stream_url, resource_url) = rewrite_urls
.map_or_else(|| (url, None), |(su, ru)| (su, ru.as_ref().map(String::to_string)));
let is_redirect = self.proxy_type.is_redirect(m3u_pli.item_type)
|| self
.target_options
.as_ref()
.and_then(|o| o.force_redirect.as_ref())
.is_some_and(|f| f.has_cluster(m3u_pli.item_type));
let should_rewrite_urls = if is_redirect { self.mask_redirect_url } else { true };
if should_rewrite_urls {
let stream_url = self.get_stream_url(&m3u_pli, self.include_type_in_url);
let resource_url = if self.rewrite_resource {
Some(self.get_resource_url(&m3u_pli))
} else {
None
};
m3u_pli.t_stream_url = stream_url;
m3u_pli.t_resource_url = resource_url;
} else {
// Keep original URL (clone required because target field is distinct)
m3u_pli.t_stream_url = m3u_pli.url.clone();
m3u_pli.t_resource_url = None;
}
m3u_pli.t_stream_url.clone_from(&stream_url);
m3u_pli.t_resource_url.clone_from(&resource_url);
(m3u_pli, has_next)
})
}
backend/src/repository/m3u_repository.rs
+13 -3
View File
@@ -15,8 +15,9 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use log::error;
use tokio::fs;
use tokio::io::{AsyncWriteExt, BufWriter as AsyncBufWriter};
use tokio::io::{AsyncWriteExt};
use tokio::task;
use crate::utils::{async_file_writer, IO_BUFFER_SIZE};
macro_rules! cant_write_result {
($path:expr, $err:expr) => {
@@ -53,13 +54,22 @@ async fn persist_m3u_playlist_as_text(
let Some(m3u_filename) = utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) else { return Ok(()); };
let file = await_playlist_write!(fs::File::create(&m3u_filename), "Can't write m3u plain playlist {} - {}", m3u_filename.display());
let mut writer = AsyncBufWriter::new(file);
// Larger buffer for sequential writes to reduce syscalls
let mut writer = async_file_writer(file);
await_playlist_write!(writer.write_all(b"#EXTM3U\n"), "Failed to write header to {} - {}", m3u_filename.display());
let mut write_counter = 0usize;
for m3u in m3u_playlist.iter() {
let line = m3u.to_m3u(target.options.as_ref(), false);
await_playlist_write!(writer.write_all(line.as_bytes()), "Failed to write entry to {} - {}", m3u_filename.display());
let bytes = line.as_bytes();
await_playlist_write!(writer.write_all(bytes), "Failed to write entry to {} - {}", m3u_filename.display());
await_playlist_write!(writer.write_all(b"\n"), "Failed to write newline to {} - {}", m3u_filename.display());
write_counter += bytes.len() + 1;
if write_counter >= IO_BUFFER_SIZE {
await_playlist_write!(writer.flush(), "Failed to flush {} - {}", m3u_filename.display());
write_counter = 0;
}
}
await_playlist_write!(writer.flush(), "Failed to flush {} - {}", m3u_filename.display());
backend/src/repository/playlist_repository.rs
-1
View File
@@ -193,7 +193,6 @@ async fn load_m3u_target_storage(app_config: &AppConfig, target: &ConfigTarget)
}
pub async fn load_playlists_into_memory_cache(app_state: &AppState) -> Result<(), TuliproxError> {
for sources in &app_state.app_config.sources.load().sources {
for target in &sources.targets {
backend/src/repository/storage.rs
+2 -1
View File
@@ -6,7 +6,8 @@ use crate::repository::storage_const;
use crate::utils;
pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf {
target_path.join(PathBuf::from(storage_const::FILE_ID_MAPPING))
// Join directly with &str to avoid an intermediate PathBuf allocation
target_path.join(storage_const::FILE_ID_MAPPING)
}
pub fn ensure_target_storage_path(cfg: &Config, target_name: &str) -> Result<PathBuf, TuliproxError> {
backend/src/repository/strm_repository.rs
+19 -7
View File
@@ -8,7 +8,7 @@ use crate::repository::storage::{ensure_target_storage_path, get_input_storage_p
use crate::repository::storage_const;
use crate::repository::xtream_repository::{xtream_get_record_file_path, InputVodInfoRecord};
use shared::utils::{extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS};
use crate::utils::{normalize_string_path, truncate_filename, FileReadGuard};
use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, FileReadGuard, IO_BUFFER_SIZE};
use chrono::Datelike;
use filetime::{set_file_times, FileTime};
use log::{error, trace};
@@ -18,7 +18,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::fs::{create_dir_all, remove_dir, remove_file, File};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader, BufWriter};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
use shared::model::{ClusterFlags, FieldGetAccessor, PlaylistGroup, PlaylistItem, PlaylistItemType, StrmExportStyle, UUIDType};
use crate::utils;
// Import the new MediaQuality struct
@@ -932,22 +932,34 @@ async fn write_strm_index_file(
let file = File::create(index_file_path)
.await
.map_err(|err| format!("Failed to create strm index file: {} {err}", index_file_path.display()))?;
let mut writer = BufWriter::new(file);
// Use a larger buffered writer for sequential writes to reduce syscalls
let mut writer = async_file_writer(file);
let mut write_counter = 0usize;
let new_line = "\n".as_bytes();
for entry in entries {
let bytes = entry.as_bytes();
write_counter += bytes.len() + 1;
writer
.write_all(entry.as_bytes())
.write_all(bytes)
.await
.map_err(|err| format!("Failed to write strm index entry: {err}"))?;
writer
.write(new_line)
.write_all(new_line)
.await
.map_err(|err| format!("Failed to write strm index entry: {err}"))?;
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
writer.flush().await.map_err(|err| format!("Failed to flush: {err}"))?;
}
}
writer
.flush()
.await
.map_err(|err| format!("failed to write strm index entry: {err}"))?;
writer
.shutdown()
.await
.map_err(|err| format!("failed to write strm index entry: {err}"))?;
Ok(())
}
@@ -989,7 +1001,7 @@ async fn write_strm_file(
async fn has_strm_file_same_hash(file_path: &PathBuf, content_hash: UUIDType) -> bool {
if let Ok(file) = File::open(&file_path).await {
let mut reader = BufReader::new(file);
let mut reader = async_file_reader(file);
let mut buffer = Vec::new();
match reader.read_to_end(&mut buffer).await {
Ok(_) => {
@@ -1018,7 +1030,7 @@ fn get_credentials_and_server_info(
async fn read_strm_file_index(strm_file_index_path: &Path) -> std::io::Result<HashSet<String>> {
let file = File::open(strm_file_index_path).await?;
let reader = BufReader::new(file);
let reader = async_file_reader(file);
let mut result = HashSet::new();
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
backend/src/repository/xtream_playlist_iterator.rs
+31 -23
View File
@@ -1,21 +1,22 @@
use std::collections::HashSet;
use log::error;
use serde_json::Value;
use shared::model::{TargetType, XtreamCluster, XtreamPlaylistItem};
use shared::error::info_err;
use shared::error::{TuliproxError};
use crate::model::{xtream_playlistitem_to_document, AppConfig, ProxyUserCredentials};
use crate::model::{ConfigTarget};
use crate::model::XtreamMappingOptions;
use crate::repository::indexed_document::{IndexedDocumentIterator};
use crate::model::{xtream_playlistitem_to_document, AppConfig, ProxyUserCredentials};
use crate::model::ConfigTarget;
use crate::repository::indexed_document::IndexedDocumentIterator;
use crate::repository::user_repository::user_get_bouquet_filter;
use crate::repository::xtream_repository::{xtream_get_file_paths, xtream_get_storage_path};
use crate::utils::FileReadGuard;
use log::error;
use serde_json::Value;
use shared::error::info_err;
use shared::error::TuliproxError;
use shared::model::{TargetType, XtreamCluster, XtreamPlaylistItem};
use std::collections::HashSet;
pub struct XtreamPlaylistIterator {
reader: IndexedDocumentIterator<u32, XtreamPlaylistItem>,
options: XtreamMappingOptions,
filter: Option<HashSet<String>>,
// Use parsed numeric filter to avoid per-item String allocations (no to_string per check)
filter_ids: Option<HashSet<u32>>,
base_url: String,
user: ProxyUserCredentials,
lookup_item: Option<(XtreamPlaylistItem, bool)>, // this is for filtered iteration
@@ -49,11 +50,20 @@ impl XtreamPlaylistIterator {
let server_info = app_config.get_user_server_info(user);
let filter = user_get_bouquet_filter(&config, &user.username, category_id, TargetType::Xtream, cluster).await;
// Parse bouquet filter (strings) once into u32 set to minimize per-item allocations
let filter_ids: Option<HashSet<u32>> = filter.as_ref().map(|set| {
set.iter().filter_map(|s| {
s.parse::<u32>().map_err(|e| {
error!("Failed to parse bouquet filter id '{s}': {e}");
e
}).ok()
}).collect()
});
Ok(Self {
reader,
options,
filter,
filter_ids,
_file_lock: file_lock,
base_url: server_info.get_base_url(),
user: user.clone(),
@@ -69,16 +79,16 @@ impl XtreamPlaylistIterator {
error!("Could not deserialize xtream item: {}", self.reader.get_path().display());
return None;
}
if let Some(set) = &self.filter {
if let Some(set) = &self.filter_ids {
if let Some((current_item, _)) = self.lookup_item.take() {
let next_valid = self.reader.find(|(pli, _)| set.contains(&pli.category_id.to_string()));
let next_valid = self.reader.find(|(pli, _)| set.contains(&pli.category_id));
self.lookup_item = next_valid;
let has_next = self.lookup_item.is_some();
Some((current_item, has_next))
} else {
let current_item = self.reader.find(|(item, _)| set.contains(&item.category_id.to_string()));
let current_item = self.reader.find(|(item, _)| set.contains(&item.category_id));
if let Some((item, _)) = current_item {
self.lookup_item = self.reader.find(|(item, _)| set.contains(&item.category_id.to_string()));
self.lookup_item = self.reader.find(|(item, _)| set.contains(&item.category_id));
let has_next = self.lookup_item.is_some();
Some((item, has_next))
} else {
@@ -89,7 +99,6 @@ impl XtreamPlaylistIterator {
self.reader.next()
}
}
}
impl Iterator for XtreamPlaylistIterator {
@@ -105,12 +114,12 @@ pub struct XtreamPlaylistJsonIterator {
}
impl XtreamPlaylistJsonIterator {
pub async fn new(
cluster: XtreamCluster,
config: &AppConfig,
target: &ConfigTarget,
category_id: Option<u32>,
user: &ProxyUserCredentials,
pub async fn new(
cluster: XtreamCluster,
config: &AppConfig,
target: &ConfigTarget,
category_id: Option<u32>,
user: &ProxyUserCredentials,
) -> Result<Self, TuliproxError> {
Ok(Self {
inner: XtreamPlaylistIterator::new(cluster, config, target, category_id, user).await?
@@ -122,7 +131,6 @@ pub fn to_doc(pli: &XtreamPlaylistItem, url: &str, options: &XtreamMappingOption
xtream_playlistitem_to_document(pli, url, options, user)
}
impl Iterator for XtreamPlaylistJsonIterator {
type Item = (String, bool);
fn next(&mut self) -> Option<Self::Item> {
backend/src/repository/xtream_repository.rs
+4 -7
View File
@@ -1076,15 +1076,12 @@ where
{
match channels {
Some((_, chans)) => {
// Convert iterator items to Result<Bytes, String>
// Convert iterator items to Result<Bytes, String> with minimal allocations
let mapped = chans.map(move |(item, has_next)| {
match serde_json::to_string(&item) {
Ok(content) => {
Ok(Bytes::from(if has_next {
format!("{content},")
} else {
content
}))
Ok(mut content) => {
if has_next { content.push(','); }
Ok(Bytes::from(content))
}
Err(_) => Ok(Bytes::from("")),
}
backend/src/utils/compression/compressed_file_reader.rs
+1 -1
View File
@@ -26,7 +26,7 @@ impl CompressedFileReader {
};
Ok(Self {
reader: BufReader::new(reader),
reader: file_reader(reader),
})
}
}
backend/src/utils/compression/compressed_file_reader_async.rs
+11 -31
View File
@@ -1,38 +1,34 @@
use crate::utils::async_file_reader;
use crate::utils::compression::compression_utils::{is_deflate, is_gzip};
use async_compression::tokio::bufread::{GzipDecoder, ZlibDecoder};
use std::path::Path;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::fs::File;
use tokio::io::{
self, AsyncRead, BufReader, AsyncSeekExt, AsyncReadExt, ReadBuf,
self, AsyncRead, AsyncReadExt, AsyncSeekExt, ReadBuf,
};
use async_compression::tokio::bufread::{GzipDecoder, ZlibDecoder};
use crate::utils::compression::compression_utils::{is_deflate, is_gzip};
pub struct CompressedFileReaderAsync {
reader: BufReader<Box<dyn AsyncRead + Unpin + Send>>,
reader: Box<dyn AsyncRead + Unpin + Send>,
}
impl CompressedFileReaderAsync {
pub async fn new(path: &Path) -> std::io::Result<Self> {
let file: File = tokio::fs::File::open(path).await?;
let mut buffered_file = BufReader::new(file);
let mut buffered_file = async_file_reader(file);
let mut header = [0u8; 2];
buffered_file.read_exact(&mut header).await?;
buffered_file.seek(io::SeekFrom::Start(0)).await?;
let reader: Box<dyn AsyncRead + Unpin + Send> = if is_gzip(&header) {
Box::new(GzipDecoder::new(buffered_file))
if is_gzip(&header) {
Ok(Self { reader: Box::new(GzipDecoder::new(buffered_file)) })
} else if is_deflate(&header) {
Box::new(ZlibDecoder::new(buffered_file))
Ok(Self { reader: Box::new(ZlibDecoder::new(buffered_file)) })
} else {
Box::new(buffered_file)
};
Ok(Self {
reader: BufReader::new(reader),
})
Ok(Self { reader: Box::new(buffered_file) })
}
}
}
@@ -46,19 +42,3 @@ impl AsyncRead for CompressedFileReaderAsync {
}
}
//
// impl AsyncBufRead for CompressedFileReaderAsync {
// fn poll_fill_buf(
// self: Pin<&mut Self>,
// cx: &mut Context<'_>,
// ) -> Poll<io::Result<&[u8]>> {
// unsafe {
// let this = self.get_unchecked_mut();
// Pin::new_unchecked(&mut this.reader).poll_fill_buf(cx)
// }
// }
//
// fn consume(mut self: Pin<&mut Self>, amt: usize) {
// Pin::new(&mut self.reader).consume(amt)
// }
// }
backend/src/utils/file/config_reader.rs
+2 -2
View File
@@ -13,7 +13,7 @@ use shared::model::{ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAlias
use shared::utils::{CONSTANTS};
use std::env;
use std::fs::File;
use std::io::{self, BufReader, Read};
use std::io::{self, Read};
use std::path::PathBuf;
use std::sync::Arc;
use arc_swap::access::{Access};
@@ -40,7 +40,7 @@ pub fn config_file_reader(file: File, resolve_env: bool) -> impl Read
if resolve_env {
EitherReader::Left(EnvResolvingReader::new(file_reader(file)))
} else {
EitherReader::Right(BufReader::new(file))
EitherReader::Right(file_reader(file))
}
}
backend/src/utils/file/csv_input_reader.rs
+6 -6
View File
@@ -189,8 +189,8 @@ pub fn get_csv_file_path(file_uri: &str) -> Result<PathBuf, Error> {
#[cfg(test)]
mod tests {
use crate::utils::file::csv_input_reader::csv_read_inputs_from_reader;
use crate::utils::resolve_env_var;
use std::io::{BufReader, Cursor};
use crate::utils::{file_reader, resolve_env_var};
use std::io::{Cursor};
use shared::model::InputType;
const M3U_BATCH: &str = r"
@@ -209,7 +209,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34
#[test]
fn test_read_inputs_xtream_as_m3u() {
let reader = BufReader::new(Cursor::new(XTREAM_BATCH));
let reader = file_reader(Cursor::new(XTREAM_BATCH));
let result = csv_read_inputs_from_reader(InputType::M3uBatch, reader);
assert!(result.is_ok());
let aliases = result.unwrap();
@@ -221,7 +221,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34
#[test]
fn test_read_inputs_m3u_as_m3u() {
let reader = BufReader::new(Cursor::new(M3U_BATCH));
let reader = file_reader(Cursor::new(M3U_BATCH));
let result = csv_read_inputs_from_reader(InputType::M3uBatch, reader);
assert!(result.is_ok());
let aliases = result.unwrap();
@@ -233,7 +233,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34
#[test]
fn test_read_inputs_xtream_as_xtream() {
let reader = BufReader::new(Cursor::new(XTREAM_BATCH));
let reader = file_reader(Cursor::new(XTREAM_BATCH));
let result = csv_read_inputs_from_reader(InputType::XtreamBatch, reader);
assert!(result.is_ok());
let aliases = result.unwrap();
@@ -245,7 +245,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34
#[test]
fn test_read_inputs_m3u_as_xtream() {
let reader = BufReader::new(Cursor::new(M3U_BATCH));
let reader = file_reader(Cursor::new(M3U_BATCH));
let result = csv_read_inputs_from_reader(InputType::XtreamBatch, reader);
assert!(result.is_ok());
let aliases = result.unwrap();
backend/src/utils/file/file_utils.rs
+24 -8
View File
@@ -1,10 +1,9 @@
use std::borrow::Cow;
use std::collections::HashSet;
use std::fs::{File, OpenOptions};
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use std::{env, fs};
use std::io::Read;
use shared::error::str_to_io_error;
use crate::utils::debug_if_enabled;
use shared::utils::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, MAPPING_FILE, SOURCE_FILE, USER_FILE};
@@ -12,20 +11,37 @@ use log::{debug, error};
use path_clean::PathClean;
use tokio::fs as tokio_fs;
pub fn file_writer<W>(w: W) -> BufWriter<W>
pub const IO_BUFFER_SIZE: usize = 256*1024; // 256kb
pub fn file_writer<W>(w: W) -> std::io::BufWriter<W>
where
W: Write,
W: std::io::Write,
{
BufWriter::with_capacity(131_072, w)
std::io::BufWriter::with_capacity(IO_BUFFER_SIZE, w)
}
pub fn file_reader<R>(r: R) -> BufReader<R>
pub fn file_reader<R>(r: R) -> std::io::BufReader<R>
where
R: Read,
R: std::io::Read,
{
BufReader::with_capacity(131_072, r)
std::io::BufReader::with_capacity(IO_BUFFER_SIZE, r)
}
pub fn async_file_writer<W>(w: W) -> tokio::io::BufWriter<W>
where
W: tokio::io::AsyncWrite,
{
tokio::io::BufWriter::with_capacity(IO_BUFFER_SIZE, w)
}
pub fn async_file_reader<R>(r: R) -> tokio::io::BufReader<R>
where
R: tokio::io::AsyncRead,
{
tokio::io::BufReader::with_capacity(IO_BUFFER_SIZE, r)
}
pub fn get_exe_path() -> PathBuf {
let default_path = std::path::PathBuf::from("./");
let current_exe = std::env::current_exe();
backend/src/utils/geoip.rs
+2 -2
View File
@@ -98,15 +98,15 @@ mod test {
use crate::utils::geoip::GeoIp;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
use crate::utils::file_reader;
#[test]
pub fn test_csv() {
let db_file = PathBuf::from("/projects/m3u-test/asn-country-ipv4.db");
let source = PathBuf::from("/projects/m3u-test/asn-country-ipv4.csv");
let file = File::open(source).expect("Could not open csv file");
let reader = BufReader::new(file);
let reader = file_reader(file);
let mut geo_ip = GeoIp::new();
let _ = geo_ip.import_ipv4_from_csv(reader, &db_file).expect("Could not import csv");
backend/src/utils/json_utils.rs
+17 -2
View File
@@ -1,4 +1,4 @@
use crate::utils::file_reader;
use crate::utils::{file_reader, file_writer};
use serde::Serialize;
use serde_json::Value;
use std::collections::{HashMap, HashSet};
@@ -49,8 +49,23 @@ where
T: Serialize,
{
let file = std::fs::File::create(path)?;
let mut buf_writer = std::io::BufWriter::new(file);
let mut buf_writer = file_writer(file);
serde_json::to_writer(&mut buf_writer, value)?;
buf_writer.flush()?;
buf_writer.into_inner()?.sync_all()
}
// pub async fn is_valid_json_file(path: &str) -> std::io::Result<bool> {
// if let Ok(file) = tokio::fs::File::open(path).await {
// let reader = async_file_reader(file);
// let stream = serde_json::Deserializer::from_reader(reader).into_iter::<serde_json::Value>();
// for item in stream {
// if item.is_err() {
// return Ok(false);
// }
// }
// Ok(true)
// } else {
// Ok(false)
// }
// }
backend/src/utils/network/epg.rs
+4 -5
View File
@@ -5,7 +5,6 @@ use crate::utils::{add_prefix_to_filename, cleanup_unlisted_files_with_suffix, p
use crate::utils::request;
use log::debug;
use std::path::PathBuf;
use std::sync::Arc;
use shared::utils::{sanitize_sensitive_info, short_hash};
fn get_input_raw_epg_file_path(url: &str, input: &ConfigInput, working_dir: &str) -> Option<PathBuf> {
@@ -14,13 +13,13 @@ fn get_input_raw_epg_file_path(url: &str, input: &ConfigInput, working_dir: &str
.map(|path| add_prefix_to_filename(&path, format!("{file_prefix}_epg_").as_str(), Some("xml")))
}
async fn download_epg_file(url: &str, client: &Arc<reqwest::Client>, input: &ConfigInput, working_dir: &str) -> Result<PathBuf, TuliproxError> {
async fn download_epg_file(url: &str, client: &reqwest::Client, input: &ConfigInput, working_dir: &str) -> Result<PathBuf, TuliproxError> {
debug!("Getting epg file path for url: {}", sanitize_sensitive_info(url));
let persist_file_path = get_input_raw_epg_file_path(url, input, working_dir);
request::get_input_epg_content_as_file(Arc::clone(client), input, working_dir, url, persist_file_path).await
request::get_input_epg_content_as_file(client, input, working_dir, url, persist_file_path).await
}
pub async fn get_xmltv(client: Arc<reqwest::Client>, input: &ConfigInput, working_dir: &str) -> (Option<TVGuide>, Vec<TuliproxError>) {
pub async fn get_xmltv(client: &reqwest::Client, input: &ConfigInput, working_dir: &str) -> (Option<TVGuide>, Vec<TuliproxError>) {
match &input.epg {
None => (None, vec![]),
Some(epg_config) => {
@@ -29,7 +28,7 @@ pub async fn get_xmltv(client: Arc<reqwest::Client>, input: &ConfigInput, workin
let mut stored_file_paths = vec![];
for epg_source in &epg_config.sources {
match download_epg_file(&epg_source.url, &client, input, working_dir).await {
match download_epg_file(&epg_source.url, client, input, working_dir).await {
Ok(file_path) => {
stored_file_paths.push(file_path.clone());
file_paths.push(PersistedEpgSource {file_path, priority: epg_source.priority, logo_override: epg_source.logo_override});
backend/src/utils/network/ip_checker.rs
+3 -5
View File
@@ -1,11 +1,9 @@
use shared::error::{TuliproxError, TuliproxErrorKind};
use crate::model::IpCheckConfig;
use regex::Regex;
use reqwest::Client;
use std::sync::Arc;
use shared::utils::sanitize_sensitive_info;
async fn fetch_ip(client: &Arc<Client>, url: &str, regex: Option<&Regex>) -> Result<String, TuliproxError> {
async fn fetch_ip(client: &reqwest::Client, url: &str, regex: Option<&Regex>) -> Result<String, TuliproxError> {
let response = client.get(url).send().await
.map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("Failed to request {}: {e}", sanitize_sensitive_info(url))))?;
@@ -28,7 +26,7 @@ async fn fetch_ip(client: &Arc<Client>, url: &str, regex: Option<&Regex>) -> Res
}
/// Fetch both IPs from a shared URL (if both regex patterns are available)
async fn fetch_combined_ips(client: &Arc<Client>, config: &IpCheckConfig, url: &str) -> (Option<String>, Option<String>) {
async fn fetch_combined_ips(client: &reqwest::Client, config: &IpCheckConfig, url: &str) -> (Option<String>, Option<String>) {
let response = client.get(url).send().await.ok();
let text = match response {
Some(r) => r.text().await.ok(),
@@ -55,7 +53,7 @@ async fn fetch_combined_ips(client: &Arc<Client>, config: &IpCheckConfig, url: &
}
/// Fetch both IPv4 and IPv6 addresses, using separate or combined URL(s)
pub async fn get_ips(client: &Arc<Client>, config: &IpCheckConfig) -> Result<(Option<String>, Option<String>), TuliproxError> {
pub async fn get_ips(client: &reqwest::Client, config: &IpCheckConfig) -> Result<(Option<String>, Option<String>), TuliproxError> {
match (&config.url_ipv4, &config.url_ipv6, &config.url) {
// Both dedicated URLs provided
(Some(url_v4), Some(url_v6), _) => {
backend/src/utils/network/m3u.rs
+4 -4
View File
@@ -6,7 +6,7 @@ use crate::processing::parser::m3u;
use crate::utils::prepare_file_path;
use crate::utils::request;
pub async fn get_m3u_playlist(client: &Arc<reqwest::Client>, cfg: &Arc<Config>, input: &Arc<ConfigInput>, working_dir: &str) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
pub async fn get_m3u_playlist(client: &reqwest::Client, cfg: &Arc<Config>, input: &Arc<ConfigInput>, working_dir: &str) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let input_source: InputSource = {
match input.staged.as_ref() {
None => input.as_ref().into(),
@@ -14,9 +14,9 @@ pub async fn get_m3u_playlist(client: &Arc<reqwest::Client>, cfg: &Arc<Config>,
}
};
let persist_file_path = prepare_file_path(input.persist.as_deref(), working_dir, "");
match request::get_input_text_content(client, &input_source, working_dir, persist_file_path).await {
Ok(text) => {
(m3u::parse_m3u(cfg, input, text.lines()), vec![])
match request::get_input_text_content_as_stream(client, &input_source, working_dir, persist_file_path).await {
Ok(reader) => {
(m3u::parse_m3u(cfg, input, reader).await, vec![])
}
Err(err) => (vec![], vec![err])
}
backend/src/utils/network/request.rs
+176 -52
View File
@@ -1,29 +1,30 @@
use futures::{StreamExt, TryStreamExt};
use log::{debug, error, log_enabled, trace, Level};
use reqwest::header::CONTENT_ENCODING;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::time::{Duration, Instant};
use futures::{StreamExt, TryStreamExt};
use log::{debug, error, log_enabled, trace, Level};
use reqwest::header::CONTENT_ENCODING;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use tokio::fs::File;
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader, BufWriter};
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio_util::io::StreamReader;
use url::Url;
use crate::api::model::persist_pipe_stream::tee_dyn_reader;
use crate::model::ConfigInput;
use crate::model::{format_elapsed_time, AppConfig, InputSource, ReverseProxyDisabledHeaderConfig};
use crate::repository::storage::get_input_storage_path;
use crate::repository::storage_const;
use crate::utils::compression::compression_utils::{is_deflate, is_gzip};
use crate::utils::{async_file_reader, async_file_writer, debug_if_enabled, IO_BUFFER_SIZE};
use crate::utils::{get_file_path, persist_file};
use shared::error::create_tuliprox_error_result;
use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind};
use shared::model::{InputFetchMethod, DEFAULT_USER_AGENT};
use crate::model::{format_elapsed_time, AppConfig, InputSource, ReverseProxyDisabledHeaderConfig};
use crate::model::{ConfigInput};
use crate::repository::storage::{get_input_storage_path};
use crate::repository::storage_const;
use crate::utils::compression::compression_utils::{is_deflate, is_gzip};
use crate::utils::{debug_if_enabled};
use shared::utils::{filter_request_header, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP};
use crate::utils::{get_file_path, persist_file};
use shared::utils::{filter_request_header, human_readable_byte_size, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum MimeCategory {
@@ -53,7 +54,7 @@ pub fn classify_content_type(headers: &[(String, String)]) -> MimeCategory {
})
}
pub async fn get_input_epg_content_as_file(client: Arc<reqwest::Client>, input: &ConfigInput, working_dir: &str, url_str: &str, persist_filepath: Option<PathBuf>) -> Result<PathBuf, TuliproxError> {
pub async fn get_input_epg_content_as_file(client: &reqwest::Client, input: &ConfigInput, working_dir: &str, url_str: &str, persist_filepath: Option<PathBuf>) -> Result<PathBuf, TuliproxError> {
debug_if_enabled!("getting input epg content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(url_str));
if url_str.parse::<url::Url>().is_ok() {
match download_epg_content_as_file(client, input, url_str, working_dir, persist_filepath).await {
@@ -96,11 +97,11 @@ pub async fn get_input_epg_content_as_file(client: Arc<reqwest::Client>, input:
}
pub async fn get_input_text_content(client: &Arc<reqwest::Client>, input: &InputSource, working_dir: &str, persist_filepath: Option<PathBuf>) -> Result<String, TuliproxError> {
pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSource, working_dir: &str, persist_filepath: Option<PathBuf>) -> Result<String, TuliproxError> {
debug_if_enabled!("getting input text content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(&input.url));
if input.url.parse::<url::Url>().is_ok() {
match download_text_content(Arc::clone(client), None, input, None, persist_filepath).await {
match download_text_content(client, None, input, None, persist_filepath).await {
Ok((content, _response_url)) => Ok(content),
Err(e) => {
error!("Failed to download input '{}': {}", &input.name, sanitize_sensitive_info(e.to_string().as_str()));
@@ -139,13 +140,58 @@ pub async fn get_input_text_content(client: &Arc<reqwest::Client>, input: &Input
}
}
pub async fn get_input_text_content_as_stream(client: &reqwest::Client, input: &InputSource, working_dir: &str, persist_filepath: Option<PathBuf>) -> Result<DynReader, TuliproxError> {
debug_if_enabled!("getting input text content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(&input.url));
if input.url.parse::<url::Url>().is_ok() {
match download_text_content_as_stream(client, None, input, None, persist_filepath).await {
Ok((content, _response_url)) => Ok(content),
Err(e) => {
error!("Failed to download input '{}': {}", &input.name, sanitize_sensitive_info(e.to_string().as_str()));
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to download")
}
}
} else {
let result = match get_file_path(working_dir, Some(PathBuf::from(&input.url))) {
Some(filepath) => {
if filepath.exists() {
match get_local_file_content_as_stream(&filepath).await {
Ok(content) => {
if let Some(path) = persist_filepath {
let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| {
debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64));
}))).await;
Some(tee)
} else {
Some(content)
}
}
Err(err) => {
return create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed : {}", err);
}
}
} else {
None
}
}
None => None
};
result.map_or_else(|| {
let msg = format!("cant read input url: {}", sanitize_sensitive_info(&input.url));
error!("{msg}");
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "{msg}")
}, Ok)
}
}
pub fn get_client_request<S: ::std::hash::BuildHasher + Default>
(client: &Arc<reqwest::Client>,
method: InputFetchMethod,
headers: Option<&HashMap<String, String, S>>,
url: &Url,
custom_headers: Option<&HashMap<String, Vec<u8>, S>>,
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> reqwest::RequestBuilder {
(client: &reqwest::Client,
method: InputFetchMethod,
headers: Option<&HashMap<String, String, S>>,
url: &Url,
custom_headers: Option<&HashMap<String, Vec<u8>, S>>,
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> reqwest::RequestBuilder {
let request = match method {
InputFetchMethod::GET => client.get(url.clone()),
InputFetchMethod::POST => {
@@ -181,7 +227,7 @@ pub fn get_request_headers<S: ::std::hash::BuildHasher + Default>(request_header
for (key, value) in custom {
let key_lc = key.to_lowercase();
if filter_request_header(key_lc.as_str()) {
if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key)) {
if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key_lc.as_str())) {
continue;
}
if header_keys.contains(key_lc.as_str()) {
@@ -210,7 +256,7 @@ pub async fn get_local_file_content(file_path: &Path) -> Result<String, std::io:
std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display()))
})?;
let mut buf_reader = BufReader::new(file);
let mut buf_reader = async_file_reader(file);
// Peek die ersten 2 Bytes, um gzip zu erkennen
let buffer = buf_reader.fill_buf().await?;
@@ -234,6 +280,26 @@ pub async fn get_local_file_content(file_path: &Path) -> Result<String, std::io:
Ok(decoded)
}
pub async fn get_local_file_content_as_stream(file_path: &Path) -> Result<DynReader, std::io::Error> {
// open file
let file = File::open(file_path).await.map_err(|err| {
std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display()))
})?;
let mut buf_reader = async_file_reader(file);
// Peek first 2 Bytes, for gzip detection
let buffer = buf_reader.fill_buf().await?;
let is_gzipped = buffer.len() >= 2 && is_gzip(&buffer[0..2]);
if is_gzipped {
// use Async Gzip Decoder
Ok(Box::pin(async_compression::tokio::bufread::GzipDecoder::new(buf_reader)))
} else {
Ok(Box::pin(buf_reader))
}
}
// pub fn get_local_file_content_blocking(file_path: &PathBuf) -> Result<String, Error> {
// match fs::read(file_path) {
// Ok(content) => decode_local_file_bytes(content).await,
@@ -242,28 +308,37 @@ pub async fn get_local_file_content(file_path: &Path) -> Result<String, std::io:
// }
async fn get_remote_content_as_file(client: Arc<reqwest::Client>, input: &ConfigInput, url: &Url, file_path: &Path) -> Result<PathBuf, std::io::Error> {
async fn get_remote_content_as_file(client: &reqwest::Client, input: &ConfigInput, url: &Url, file_path: &Path) -> Result<PathBuf, std::io::Error> {
let start_time = Instant::now();
let request = get_client_request(&client, input.method, Some(&input.headers), url, None, None);
let request = get_client_request(client, input.method, Some(&input.headers), url, None, None);
match request.send().await {
Ok(response) => {
if response.status().is_success() {
// Open a file in write mode
let mut file = BufWriter::with_capacity(8192, File::create(file_path).await?);
let mut writer = async_file_writer(File::create(file_path).await?);
let mut write_counter = 0;
// Stream the response body in chunks
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
match chunk {
Ok(bytes) => {
file.write_all(&bytes).await?;
write_counter += bytes.len();
writer.write_all(&bytes).await?;
if write_counter >= IO_BUFFER_SIZE {
writer.flush().await?;
write_counter = 0;
}
}
Err(err) => {
let _ = writer.flush().await;
let _ = writer.shutdown().await;
return Err(str_to_io_error(&format!("Failed to read chunk: {err}")));
}
}
}
file.flush().await?;
writer.flush().await?;
writer.shutdown().await?;
let elapsed = start_time.elapsed().as_secs();
debug!("File downloaded successfully to {}, took:{}", file_path.display(), format_elapsed_time(elapsed));
Ok(file_path.to_path_buf())
@@ -275,16 +350,23 @@ async fn get_remote_content_as_file(client: Arc<reqwest::Client>, input: &Config
}
}
type DynReader = Pin<Box<dyn AsyncRead + Send>>;
pub type DynReader = Pin<Box<dyn AsyncRead + Send>>;
#[allow(clippy::implicit_hasher)]
pub async fn get_remote_content_as_stream(
client: Arc<reqwest::Client>,
client: &reqwest::Client,
input: &InputSource,
headers: Option<&HeaderMap>,
url: &Url,
method: InputFetchMethod,
headers: Option<&HashMap<String, String>>
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>,
) -> Result<(DynReader, String), Error> {
let request = get_client_request(&client, method, headers, url, None, None);
let custom_headers = headers.map(|h| {
h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::<HashMap<_, _>>()
});
let merged = get_request_headers(Some(&input.headers), custom_headers.as_ref(), disabled_headers);
let headers: HashMap<String, String> = merged.iter().map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect();
let request = get_client_request(client, input.method, Some(&headers), url, None, None);
let response = request.send().await.map_err(std::io::Error::other)?;
if !response.status().is_success() {
@@ -298,7 +380,7 @@ pub async fn get_remote_content_as_stream(
let stream_reader = StreamReader::new(
response.bytes_stream().map_err(std::io::Error::other),
);
let mut buf_reader = BufReader::new(stream_reader);
let mut buf_reader = async_file_reader(stream_reader);
let peek = buf_reader.fill_buf().await?;
if peek.len() >= 2 {
@@ -320,22 +402,17 @@ pub async fn get_remote_content_as_stream(
Ok((reader, response_url))
}
async fn get_remote_content(client: Arc<reqwest::Client>, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> Result<(String, String), Error> {
async fn get_remote_content(client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> Result<(String, String), Error> {
let start_time = Instant::now();
let custom_headers = headers.map(|h| {
h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::<HashMap<_, _>>()});
let merged = get_request_headers(Some(&input.headers), custom_headers.as_ref(), disabled_headers);
let headers: HashMap<String, String> = merged.iter().map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect();
let (mut stream, response_url) = get_remote_content_as_stream(client.clone(), url, input.method, Some(&headers)).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
let (mut stream, response_url) = get_remote_content_as_stream(client, input, headers, url, disabled_headers).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
let mut content = String::new();
stream.read_to_string(&mut content).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
debug_if_enabled!("Request took: {} {}", format_elapsed_time(start_time.elapsed().as_secs()), sanitize_sensitive_info(url.as_str()));
Ok((content, response_url))
}
async fn download_epg_content_as_file(client: Arc<reqwest::Client>, input: &ConfigInput, url_str: &str, working_dir: &str, persist_filepath: Option<PathBuf>) -> Result<PathBuf, Error> {
async fn download_epg_content_as_file(client: &reqwest::Client, input: &ConfigInput, url_str: &str, working_dir: &str, persist_filepath: Option<PathBuf>) -> Result<PathBuf, Error> {
if let Ok(url) = url_str.parse::<url::Url>() {
if url.scheme() == "file" {
url.to_file_path().map_or_else(|()| Err(Error::new(ErrorKind::Unsupported, format!("Unknown file {}", sanitize_sensitive_info(url_str)))), |file_path| if file_path.exists() {
@@ -361,7 +438,7 @@ async fn download_epg_content_as_file(client: Arc<reqwest::Client>, input: &Conf
}
pub async fn download_text_content(
client: Arc<reqwest::Client>,
client: &reqwest::Client,
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>,
input: &InputSource,
headers: Option<&HeaderMap>,
@@ -389,14 +466,48 @@ pub async fn download_text_content(
Err(err) => Err(err),
}
} else {
Err(str_to_io_error(&format!(
"Malformed URL {}",
sanitize_sensitive_info(&input.url)
)))
Err(str_to_io_error(&format!("Malformed URL {}", sanitize_sensitive_info(&input.url))))
}
}
async fn download_json_content(client: Arc<reqwest::Client>, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<serde_json::Value, Error> {
pub async fn download_text_content_as_stream(
client: &reqwest::Client,
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>,
input: &InputSource,
headers: Option<&HeaderMap>,
persist_filepath: Option<PathBuf>,
) -> Result<(DynReader, String), Error> {
if let Ok(url) = input.url.parse::<url::Url>() {
let result = if url.scheme() == "file" {
match url.to_file_path() {
Ok(file_path) => get_local_file_content_as_stream(&file_path).await.map(|c| (c, url.to_string())),
Err(()) => Err(str_to_io_error(&format!(
"Unknown file {}",
sanitize_sensitive_info(&input.url)
))),
}
} else {
get_remote_content_as_stream(client, input, headers, &url, disabled_headers).await
};
match result {
Ok((content, response_url)) => {
if let Some(path) = persist_filepath {
let tee_reader: DynReader = tee_dyn_reader(content, &path, Some(Arc::new(|size| {
debug!("Persisted {size} bytes");
}))).await;
Ok((tee_reader, response_url))
} else {
Ok((content, response_url))
}
}
Err(err) => Err(err),
}
} else {
Err(str_to_io_error(&format!("Malformed URL {}", sanitize_sensitive_info(&input.url))))
}
}
async fn download_json_content(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<serde_json::Value, Error> {
debug_if_enabled!("downloading json content from {}", sanitize_sensitive_info(&input.url));
match download_text_content(client, disabled_headers, input, None, persist_filepath).await {
Ok((content, _response_url)) => {
@@ -409,13 +520,27 @@ async fn download_json_content(client: Arc<reqwest::Client>, disabled_headers: O
}
}
pub async fn get_input_json_content(client: Arc<reqwest::Client>, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<serde_json::Value, TuliproxError> {
pub async fn get_input_json_content(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<serde_json::Value, TuliproxError> {
match download_json_content(client, disabled_headers, input, persist_filepath).await {
Ok(content) => Ok(content),
Err(e) => create_tuliprox_error_result!(TuliproxErrorKind::Notify, "cant download input {}, url: {} => {}", input.name, sanitize_sensitive_info(&input.url), sanitize_sensitive_info(e.to_string().as_str()))
}
}
async fn download_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<DynReader, Error> {
debug_if_enabled!("downloading json content from {}", sanitize_sensitive_info(&input.url));
match download_text_content_as_stream(client, disabled_headers, input, None, persist_filepath).await {
Ok((reader, _response_url)) => Ok(reader),
Err(err) => Err(err)
}
}
pub async fn get_input_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<DynReader, TuliproxError> {
match download_json_content_as_stream(client, disabled_headers, input, persist_filepath).await {
Ok(stream) => Ok(stream),
Err(e) => create_tuliprox_error_result!(TuliproxErrorKind::Notify, "cant download input {}, url: {} => {}", input.name, sanitize_sensitive_info(&input.url), sanitize_sensitive_info(e.to_string().as_str()))
}
}
pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder {
let config = cfg.config.load();
@@ -425,7 +550,6 @@ pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder {
.pool_max_idle_per_host(10)
.danger_accept_invalid_certs(config.accept_insecure_ssl_certificates);
if let Some(proxy_cfg) = config.proxy.as_ref() {
match Url::parse(&proxy_cfg.url) {
Ok(mut url) => {
@@ -443,7 +567,7 @@ pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder {
Ok(p) => { client = client.proxy(p); }
Err(err) => error!("Failed to create SOCKS proxy {url}: {err}"),
}
},
}
"http" | "https" => {
match reqwest::Proxy::all(url.as_str()) {
Ok(p) => {
backend/src/utils/network/xtream.rs
+12 -12
View File
@@ -46,7 +46,7 @@ pub fn get_xtream_player_api_info_url(input: &ConfigInput, cluster: XtreamCluste
}
pub async fn get_xtream_stream_info_content(client: Arc<reqwest::Client>, input: &InputSource) -> Result<String, Error> {
pub async fn get_xtream_stream_info_content(client: &reqwest::Client, input: &InputSource) -> Result<String, Error> {
match request::download_text_content(client, None, input, None, None).await {
Ok((content, _response_url)) => Ok(content),
Err(err) => Err(err)
@@ -54,7 +54,7 @@ pub async fn get_xtream_stream_info_content(client: Arc<reqwest::Client>, input:
}
#[allow(clippy::too_many_arguments)]
pub async fn get_xtream_stream_info<P>(client: Arc<reqwest::Client>,
pub async fn get_xtream_stream_info<P>(client: &reqwest::Client,
app_state: &Arc<AppState>,
user: &ProxyUserCredentials,
input: &ConfigInput,
@@ -135,12 +135,12 @@ const ACTIONS: [(XtreamCluster, &str, &str); 3] = [
(XtreamCluster::Video, crate::model::XC_ACTION_GET_VOD_CATEGORIES, crate::model::XC_ACTION_GET_VOD_STREAMS),
(XtreamCluster::Series, crate::model::XC_ACTION_GET_SERIES_CATEGORIES, crate::model::XC_ACTION_GET_SERIES)];
async fn xtream_login(cfg: &Config, client: &Arc<reqwest::Client>, input: &InputSource, username: &str) -> Result<Option<XtreamLoginInfo>, TuliproxError> {
let content = if let Ok(content) = request::get_input_json_content(Arc::clone(client), None, input, None).await {
async fn xtream_login(cfg: &Config, client: &reqwest::Client, input: &InputSource, username: &str) -> Result<Option<XtreamLoginInfo>, TuliproxError> {
let content = if let Ok(content) = request::get_input_json_content(client, None, input, None).await {
content
} else {
let input_source_account_info = input.with_url(format!("{}&action={}", &input.url, crate::model::XC_ACTION_GET_ACCOUNT_INFO));
match request::get_input_json_content(Arc::clone(client), None, &input_source_account_info, None).await {
match request::get_input_json_content(client, None, &input_source_account_info, None).await {
Ok(content) => content,
Err(err) => {
warn!("Failed to login xtream account {username} {err}");
@@ -182,7 +182,7 @@ async fn xtream_login(cfg: &Config, client: &Arc<reqwest::Client>, input: &Input
}
}
pub async fn notify_account_expire(exp_date: Option<i64>, cfg: &Config, client: &Arc<reqwest::Client>, username: &str, input_name: &str) {
pub async fn notify_account_expire(exp_date: Option<i64>, cfg: &Config, client: &reqwest::Client, username: &str, input_name: &str) {
if let Some(expiration_timestamp) = exp_date {
let now_secs = Utc::now().timestamp(); // UTC-Time
if expiration_timestamp > now_secs {
@@ -202,7 +202,7 @@ pub async fn notify_account_expire(exp_date: Option<i64>, cfg: &Config, client:
}
}
pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &Arc<reqwest::Client>, input: &Arc<ConfigInput>, working_dir: &str) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &reqwest::Client, input: &Arc<ConfigInput>, working_dir: &str) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let input_source: InputSource = {
match input.staged.as_ref() {
None => input.as_ref().into(),
@@ -235,14 +235,14 @@ pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &Arc<reqwest::Client
let stream_file_path = crate::utils::prepare_file_path(input.persist.as_deref(), working_dir, format!("{stream}_").as_str());
match futures::join!(
request::get_input_json_content(Arc::clone(client), None, &input_source_category, category_file_path),
request::get_input_json_content(Arc::clone(client), None, &input_source_stream, stream_file_path)
request::get_input_json_content_as_stream(client, None, &input_source_category, category_file_path),
request::get_input_json_content_as_stream(client, None, &input_source_stream, stream_file_path)
) {
(Ok(category_content), Ok(stream_content)) => {
match xtream::parse_xtream(input,
*xtream_cluster,
&category_content,
&stream_content) {
category_content,
stream_content).await {
Ok(sub_playlist_parsed) => {
if let Some(mut xtream_sub_playlist) = sub_playlist_parsed {
playlist_groups.append(&mut xtream_sub_playlist);
@@ -265,7 +265,7 @@ pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &Arc<reqwest::Client
(playlist_groups, errors)
}
async fn check_alias_user_state(cfg: &Arc<Config>, client: &Arc<reqwest::Client>, input: &Arc<ConfigInput>) {
async fn check_alias_user_state(cfg: &Arc<Config>, client: &reqwest::Client, input: &Arc<ConfigInput>) {
if let Some(aliases) = input.aliases.as_ref() {
for alias in aliases {
if is_input_expired(alias.exp_date) {
backend/src/utils/telegram.rs
+1 -2
View File
@@ -1,4 +1,3 @@
use std::sync::Arc;
use log::{debug, error};
use url::Url;
@@ -63,7 +62,7 @@ pub fn telegram_create_instance(bot_token: &str, chat_id: &str) -> BotInstance {
}
pub async fn telegram_send_message(
client: &Arc<reqwest::Client>,
client: &reqwest::Client,
instance: &BotInstance,
msg: &str,
options: Option<&SendMessageOption>,
backend/src/utils/trakt/client.rs
+2 -3
View File
@@ -1,21 +1,20 @@
use crate::model::{TraktApiConfig, TraktListConfig, TraktListItem};
use shared::error::TuliproxError;
use reqwest::header::{HeaderMap, HeaderValue};
use std::sync::Arc;
use log::{debug, info};
use shared::error::{info_err};
use shared::utils::trim_last_slash;
use super::errors::{handle_trakt_api_error};
pub struct TraktClient {
client: Arc<reqwest::Client>,
client: reqwest::Client,
api_config: TraktApiConfig,
// Pre-computed headers to avoid recreating them each time
headers: HeaderMap,
}
impl TraktClient {
pub fn new(client: Arc<reqwest::Client>, api_config: TraktApiConfig) -> Self {
pub fn new(client: reqwest::Client, api_config: TraktApiConfig) -> Self {
let headers = Self::create_headers(&api_config);
Self {
client,
frontend/Cargo.toml
+2 -2
View File
@@ -1,10 +1,10 @@
[package]
name = "frontend"
version = "3.2.17"
version = "3.2.18"
edition = "2021"
[dependencies]
shared = { version = "3.2.17", path = "../shared" }
shared = { version = "3.2.18", path = "../shared" }
chrono = "0"
yew = "0.21"
yew-router = "0.18"
shared/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "shared"
version = "3.2.17"
version = "3.2.18"
edition = "2021"
[dependencies]
shared/src/utils/serde_utils.rs
+92 -28
View File
@@ -1,7 +1,6 @@
use crate::error::to_io_error;
use chrono::{NaiveDateTime, ParseError, TimeZone, Utc};
use serde::de::DeserializeOwned;
use serde::Deserialize;
use serde::{Deserialize, Deserializer};
use serde_json::Value;
use std::io;
@@ -55,43 +54,108 @@ where
})
}
pub fn deserialize_number_from_string<'de, D, T: DeserializeOwned + std::str::FromStr>(
deserializer: D,
) -> Result<Option<T>, D::Error>
pub fn deserialize_number_from_string<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
where
D: serde::Deserializer<'de>,
D: Deserializer<'de>,
T: std::str::FromStr,
{
// we define a local enum type inside of the function
// because it is untagged, serde will deserialize as the first variant
// that it can
#[derive(Deserialize)]
#[serde(untagged)]
enum MaybeNumber<U> {
// if it can be parsed as Option<T>, it will be
Value(Option<U>),
// otherwise try parsing as a string
NumberString(String),
}
let raw: Value = Value::deserialize(deserializer)?;
// deserialize into local enum
let value: MaybeNumber<T> = Deserialize::deserialize(deserializer)?;
match value {
// if parsed as T or None, return that
MaybeNumber::Value(value) => Ok(value),
match raw {
// Null → None
Value::Null => Ok(None),
// (if it is any other string)
MaybeNumber::NumberString(s) => {
// its a number
Value::Number(n) => {
let s = n.to_string();
if let Ok(r) = s.parse::<T>() {
Ok(Some(r))
} else {
Ok(None)
}
}
// String → extract first number
Value::String(s) => {
let s = s.trim();
if s.is_empty() {
return Ok(None);
}
// parse string to number, if fails return None
if let Ok(num) = s.parse::<T>() {
return Ok(Some(num));
// Find first digit OR a '.' that is immediately followed by a digit;
// include optional sign immediately before it (ignoring whitespace).
let mut last_non_ws: Option<(usize, char)> = None;
let mut num_pos: Option<usize> = None;
for (i, c) in s.char_indices() {
if c.is_ascii_digit() {
num_pos = Some(i);
break;
}
if c == '.' && s[i + 1..].chars().next().is_some_and(|n| n.is_ascii_digit()) {
num_pos = Some(i);
break;
}
if !c.is_whitespace() {
last_non_ws = Some((i, c));
}
}
let Some(num_i) = num_pos else { return Ok(None); };
let start = match last_non_ws {
Some((i, '-')) | Some((i, '+')) => i,
_ => num_i,
};
let mut it = s[start..].chars().peekable();
// optional sign
let mut out = String::new();
if matches!(it.peek(), Some('-' | '+')) {
out.push(it.next().unwrap());
while matches!(it.peek(), Some(c) if c.is_whitespace()) {
it.next();
}
}
serde_json::from_str::<T>(s).map_or_else(|_| Ok(None), |val| Ok(Some(val)))
// digits + optional single dot
let mut saw_digit = false;
let mut saw_dot = false;
while let Some(&c) = it.peek() {
if c.is_ascii_digit() {
saw_digit = true;
out.push(c);
it.next();
continue;
}
if c == '.' && !saw_dot {
saw_dot = true;
out.push(c);
it.next();
continue;
}
break;
}
if !saw_digit {
return Ok(None);
}
// Try full parse; if it fails and we included '.', fall back to integer part.
if let Ok(v) = out.parse::<T>() {
return Ok(Some(v));
}
if saw_dot {
let int_part = out.split('.').next().unwrap_or("");
if !int_part.is_empty() && int_part != "-" && int_part != "+" {
if let Ok(v) = int_part.parse::<T>() {
return Ok(Some(v));
}
}
}
Ok(None)
}
// invalid -> return None
_ => Ok(None),
}
}
shared/src/utils/string_utils.rs
+39
View File
@@ -131,11 +131,34 @@ pub fn longest<'a>(a: &'a str, b: &'a str) -> &'a str {
if a.len() >= b.len() { a } else { b }
}
// ------------------------------------------------------------
// Generic string concatenation macro with optional capacity hint
// Usage:
// let s = concat_string!("/", user, "/", pass, "/", id);
// let s = concat_string!(cap = 128; prefix, "/", value);
// The macro writes all arguments using Display into a preallocated String
// to minimize temporary allocations and copies.
// ------------------------------------------------------------
#[macro_export]
macro_rules! concat_string {
(cap = $cap:expr; $($arg:expr),* $(,)?) => {{
let mut __s = ::std::string::String::with_capacity($cap);
$( let _ = ::std::fmt::Write::write_fmt(&mut __s, format_args!("{}", $arg)); )*
__s
}};
( $($arg:expr),* $(,)?) => {{
let mut __s = ::std::string::String::new();
$( let _ = ::std::fmt::Write::write_fmt(&mut __s, format_args!("{}", $arg)); )*
__s
}};
}
#[cfg(test)]
mod test {
use std::collections::HashSet;
use crate::utils::Capitalize;
use super::generate_random_string;
use crate as shared; // allow path-based macro call in tests
#[test]
fn test_generate_random_string() {
@@ -151,4 +174,20 @@ mod test {
assert_eq!("hELLO".capitalize(), "Hello");
}
#[test]
fn test_concat_string_basic() {
let a = "hello";
let b = String::from("world");
let n = 42;
let s = shared::concat_string!(a, " ", b, " ", n);
assert_eq!(s, "hello world 42");
}
#[test]
fn test_concat_string_with_cap() {
let part = "abc";
let s = shared::concat_string!(cap = 16; part, "/", 123);
assert_eq!(s, "abc/123");
}
}