euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-30 18:52:31 +01:00
Code Issues Packages Projects Releases 5 Wiki Activity

xtream and m3u parsing with streaming

Browse Source
This commit is contained in:
euzu
2025-12-12 15:33:41 +01:00
parent c22a3df93b
commit ee0441877d
21 changed files with 388 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/utils/network/m3u.rs
+3 -3
View File
@@ -14,9 +14,9 @@ pub async fn get_m3u_playlist(client: &reqwest::Client, cfg: &Arc<Config>, input
}
};
let persist_file_path = prepare_file_path(input.persist.as_deref(), working_dir, "");
match request::get_input_text_content(client, &input_source, working_dir, persist_file_path).await {
Ok(text) => {
(m3u::parse_m3u(cfg, input, text.lines()), vec![])
match request::get_input_text_content_as_stream(client, &input_source, working_dir, persist_file_path).await {
Ok(reader) => {
(m3u::parse_m3u(cfg, input, reader).await, vec![])
}
Err(err) => (vec![], vec![err])
}
backend/src/utils/network/request.rs
+160 -33
View File
@@ -1,28 +1,30 @@
use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::time::{Duration, Instant};
use futures::{StreamExt, TryStreamExt};
use log::{debug, error, log_enabled, trace, Level};
use reqwest::header::CONTENT_ENCODING;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::fs::File;
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio_util::io::StreamReader;
use url::Url;
use shared::error::create_tuliprox_error_result;
use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind};
use shared::model::{InputFetchMethod, DEFAULT_USER_AGENT};
use crate::model::{format_elapsed_time, AppConfig, InputSource, ReverseProxyDisabledHeaderConfig};
use crate::model::{ConfigInput};
use crate::repository::storage::{get_input_storage_path};
use crate::model::ConfigInput;
use crate::repository::storage::get_input_storage_path;
use crate::repository::storage_const;
use crate::utils::compression::compression_utils::{is_deflate, is_gzip};
use crate::utils::{async_file_reader, async_file_writer, debug_if_enabled, IO_BUFFER_SIZE};
use shared::utils::{filter_request_header, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP};
use crate::utils::{get_file_path, persist_file};
use shared::error::create_tuliprox_error_result;
use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind};
use shared::model::{InputFetchMethod, DEFAULT_USER_AGENT};
use shared::utils::{filter_request_header, human_readable_byte_size, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP};
use crate::api::model::persist_pipe_stream::tee_dyn_reader;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum MimeCategory {
@@ -138,13 +140,63 @@ pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSourc
}
}
pub async fn get_input_text_content_as_stream(client: &reqwest::Client, input: &InputSource, working_dir: &str, persist_filepath: Option<PathBuf>) -> Result<DynReader, TuliproxError> {
debug_if_enabled!("getting input text content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(&input.url));
if input.url.parse::<url::Url>().is_ok() {
match download_text_content_as_stream(client, None, input, None, persist_filepath).await {
Ok((content, _response_url)) => Ok(content),
Err(e) => {
error!("Failed to download input '{}': {}", &input.name, sanitize_sensitive_info(e.to_string().as_str()));
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to download")
}
}
} else {
let result = match get_file_path(working_dir, Some(PathBuf::from(&input.url))) {
Some(filepath) => {
if filepath.exists() {
match get_local_file_content_as_stream(&filepath).await {
Ok(content) => {
if persist_filepath.is_some() {
let tee_reader: DynReader = if let Some(path) = persist_filepath {
let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| {
debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64));
}))).await;
Box::pin(tee)
} else {
content
};
Some(tee_reader)
} else {
Some(content)
}
},
Err(err) => {
return create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed : {}", err);
}
}
} else {
None
}
}
None => None
};
result.map_or_else(|| {
let msg = format!("cant read input url: {}", sanitize_sensitive_info(&input.url));
error!("{msg}");
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "{msg}")
}, Ok)
}
}
pub fn get_client_request<S: ::std::hash::BuildHasher + Default>
(client: &reqwest::Client,
method: InputFetchMethod,
headers: Option<&HashMap<String, String, S>>,
url: &Url,
custom_headers: Option<&HashMap<String, Vec<u8>, S>>,
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> reqwest::RequestBuilder {
(client: &reqwest::Client,
method: InputFetchMethod,
headers: Option<&HashMap<String, String, S>>,
url: &Url,
custom_headers: Option<&HashMap<String, Vec<u8>, S>>,
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> reqwest::RequestBuilder {
let request = match method {
InputFetchMethod::GET => client.get(url.clone()),
InputFetchMethod::POST => {
@@ -233,6 +285,26 @@ pub async fn get_local_file_content(file_path: &Path) -> Result<String, std::io:
Ok(decoded)
}
pub async fn get_local_file_content_as_stream(file_path: &Path) -> Result<DynReader, std::io::Error> {
// open file
let file = File::open(file_path).await.map_err(|err| {
std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display()))
})?;
let mut buf_reader = async_file_reader(file);
// Peek first 2 Bytes, for gzip detection
let buffer = buf_reader.fill_buf().await?;
let is_gzipped = buffer.len() >= 2 && is_gzip(&buffer[0..2]);
if is_gzipped {
// use Async Gzip Decoder
Ok(Box::pin(async_compression::tokio::bufread::GzipDecoder::new(buf_reader)))
} else {
Ok(Box::pin(buf_reader))
}
}
// pub fn get_local_file_content_blocking(file_path: &PathBuf) -> Result<String, Error> {
// match fs::read(file_path) {
// Ok(content) => decode_local_file_bytes(content).await,
@@ -283,16 +355,23 @@ async fn get_remote_content_as_file(client: &reqwest::Client, input: &ConfigInpu
}
}
type DynReader = Pin<Box<dyn AsyncRead + Send>>;
pub type DynReader = Pin<Box<dyn AsyncRead + Send>>;
#[allow(clippy::implicit_hasher)]
pub async fn get_remote_content_as_stream(
client: &reqwest::Client,
input: &InputSource,
headers: Option<&HeaderMap>,
url: &Url,
method: InputFetchMethod,
headers: Option<&HashMap<String, String>>
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>,
) -> Result<(DynReader, String), Error> {
let request = get_client_request(client, method, headers, url, None, None);
let custom_headers = headers.map(|h| {
h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::<HashMap<_, _>>()
});
let merged = get_request_headers(Some(&input.headers), custom_headers.as_ref(), disabled_headers);
let headers: HashMap<String, String> = merged.iter().map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect();
let request = get_client_request(client, input.method, Some(&headers), url, None, None);
let response = request.send().await.map_err(std::io::Error::other)?;
if !response.status().is_success() {
@@ -331,12 +410,7 @@ pub async fn get_remote_content_as_stream(
async fn get_remote_content(client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> Result<(String, String), Error> {
let start_time = Instant::now();
let custom_headers = headers.map(|h| {
h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::<HashMap<_, _>>()});
let merged = get_request_headers(Some(&input.headers), custom_headers.as_ref(), disabled_headers);
let headers: HashMap<String, String> = merged.iter().map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect();
let (mut stream, response_url) = get_remote_content_as_stream(client, url, input.method, Some(&headers)).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
let (mut stream, response_url) = get_remote_content_as_stream(client, input, headers, url, disabled_headers).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
let mut content = String::new();
stream.read_to_string(&mut content).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
debug_if_enabled!("Request took: {} {}", format_elapsed_time(start_time.elapsed().as_secs()), sanitize_sensitive_info(url.as_str()));
@@ -397,10 +471,50 @@ pub async fn download_text_content(
Err(err) => Err(err),
}
} else {
Err(str_to_io_error(&format!(
"Malformed URL {}",
sanitize_sensitive_info(&input.url)
)))
Err(str_to_io_error(&format!("Malformed URL {}",sanitize_sensitive_info(&input.url))))
}
}
pub async fn download_text_content_as_stream(
client: &reqwest::Client,
disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>,
input: &InputSource,
headers: Option<&HeaderMap>,
persist_filepath: Option<PathBuf>,
) -> Result<(DynReader, String), Error> {
if let Ok(url) = input.url.parse::<url::Url>() {
let result = if url.scheme() == "file" {
match url.to_file_path() {
Ok(file_path) => get_local_file_content_as_stream(&file_path).await.map(|c| (c, url.to_string())),
Err(()) => Err(str_to_io_error(&format!(
"Unknown file {}",
sanitize_sensitive_info(&input.url)
))),
}
} else {
get_remote_content_as_stream(client, input, headers, &url, disabled_headers).await
};
match result {
Ok((content, response_url)) => {
if persist_filepath.is_some() {
let tee_reader: DynReader = if let Some(path) = persist_filepath {
let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| {
debug!("Persisted {size} bytes");
}))).await;
Box::pin(tee)
} else {
content
};
Ok((tee_reader, response_url))
} else {
Ok((content, response_url))
}
}
Err(err) => Err(err),
}
} else {
Err(str_to_io_error(&format!("Malformed URL {}", sanitize_sensitive_info(&input.url))))
}
}
@@ -424,6 +538,20 @@ pub async fn get_input_json_content(client: &reqwest::Client, disabled_headers:
}
}
async fn download_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<DynReader, Error> {
debug_if_enabled!("downloading json content from {}", sanitize_sensitive_info(&input.url));
match download_text_content_as_stream(client, disabled_headers, input, None, persist_filepath).await {
Ok((reader, _response_url)) => Ok(reader),
Err(err) => Err(err)
}
}
pub async fn get_input_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<DynReader, TuliproxError> {
match download_json_content_as_stream(client, disabled_headers, input, persist_filepath).await {
Ok(stream) => Ok(stream),
Err(e) => create_tuliprox_error_result!(TuliproxErrorKind::Notify, "cant download input {}, url: {} => {}", input.name, sanitize_sensitive_info(&input.url), sanitize_sensitive_info(e.to_string().as_str()))
}
}
pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder {
let config = cfg.config.load();
@@ -433,7 +561,6 @@ pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder {
.pool_max_idle_per_host(10)
.danger_accept_invalid_certs(config.accept_insecure_ssl_certificates);
if let Some(proxy_cfg) = config.proxy.as_ref() {
match Url::parse(&proxy_cfg.url) {
Ok(mut url) => {
@@ -451,7 +578,7 @@ pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder {
Ok(p) => { client = client.proxy(p); }
Err(err) => error!("Failed to create SOCKS proxy {url}: {err}"),
}
},
}
"http" | "https" => {
match reqwest::Proxy::all(url.as_str()) {
Ok(p) => {
backend/src/utils/network/xtream.rs
+4 -4
View File
@@ -235,14 +235,14 @@ pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &reqwest::Client, in
let stream_file_path = crate::utils::prepare_file_path(input.persist.as_deref(), working_dir, format!("{stream}_").as_str());
match futures::join!(
request::get_input_json_content(client, None, &input_source_category, category_file_path),
request::get_input_json_content(client, None, &input_source_stream, stream_file_path)
request::get_input_json_content_as_stream(client, None, &input_source_category, category_file_path),
request::get_input_json_content_as_stream(client, None, &input_source_stream, stream_file_path)
) {
(Ok(category_content), Ok(stream_content)) => {
match xtream::parse_xtream(input,
*xtream_cluster,
&category_content,
&stream_content) {
category_content,
stream_content).await {
Ok(sub_playlist_parsed) => {
if let Some(mut xtream_sub_playlist) = sub_playlist_parsed {
playlist_groups.append(&mut xtream_sub_playlist);