xtream and m3u parsing with streaming

backend/src/api/api_utils.rs

@@ -479,13 +479,7 @@ async fn create_stream_response_details(
         | ProviderStreamState::GracePeriod(_provider_name, request_url) => {
             let parsed_url = Url::parse(&request_url);
             let ((stream, stream_info), reconnect_flag) = if let Ok(url) = parsed_url {
-                let disabled_headers = app_state
-                    .app_config
-                    .config
-                    .load()
-                    .reverse_proxy
-                    .as_ref()
-                    .and_then(|r| r.disabled_header.clone());
+                let disabled_headers = app_state.get_disabled_headers();
                 let provider_stream_factory_options = ProviderStreamFactoryOptions::new(
                     fingerprint.addr,
                     item_type,
@@ -1135,10 +1129,7 @@ async fn fetch_resource_with_retry(
         .reverse_proxy
         .as_ref()
         .map_or_else(ResourceRetryConfig::get_default_retry_values, |rp| rp.resource_retry.get_retry_values());
-    let disabled_headers = config
-        .reverse_proxy
-        .as_ref()
-        .and_then(|r| r.disabled_header.clone());
+    let disabled_headers = app_state.get_disabled_headers();
     for attempt in 0..max_attempts {
         let client = request::get_client_request(
             &app_state.http_client.load(),

backend/src/api/endpoints/hls_api.rs

@@ -111,11 +111,7 @@ pub(in crate::api) async fn handle_hls_stream_request(
     // Don't forward Range on playlist fetch; segments use original headers in provider path
     let filter_header: HeaderFilter = Some(Box::new(|name: &str| !name.eq_ignore_ascii_case("range")));
     let forwarded = get_headers_from_request(req_headers, &filter_header);
-    let config = app_state.app_config.config.load();
-    let disabled_headers = config
-        .reverse_proxy
-        .as_ref()
-        .and_then(|r| r.disabled_header.clone());
+    let disabled_headers = app_state.get_disabled_headers();
     let headers = request::get_request_headers(None, Some(&forwarded), disabled_headers.as_ref());
     let input_source = InputSource::from(input).with_url(request_url);
     match request::download_text_content(

backend/src/api/endpoints/v1_api.rs

@@ -89,10 +89,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State<Arc<
                 method: InputFetchMethod::GET,
                 headers: HashMap::default(),
             };
-            let disabled_headers = config
-                .reverse_proxy
-                .as_ref()
-                .and_then(|r| r.disabled_header.clone());
+            let disabled_headers = app_state.get_disabled_headers();
             return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await {
                    Ok((content, _)) => {
                        let reader = Cursor::new(content);

backend/src/api/endpoints/v1_api_config.rs

@@ -118,11 +118,7 @@ async fn config_batch_content(
         // The url is changed at this point, we need the raw url for the batch file
         if let Some(batch_url) = config_input.t_batch_url.as_ref() {
             let input_source = InputSource::from(&*config_input).with_url(batch_url.to_owned());
-            let config = app_state.app_config.config.load();
-            let disabled_headers = config
-                .reverse_proxy
-                .as_ref()
-                .and_then(|r| r.disabled_header.clone());
+            let disabled_headers = app_state.get_disabled_headers();
             return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await {
                 Ok((content, _path)) => {
                     // Return CSV with explicit content-type

backend/src/api/main_api.rs

@@ -31,6 +31,7 @@ use std::sync::atomic::AtomicI8;
 use std::sync::Arc;
 use tokio_util::sync::CancellationToken;
 use tower_governor::key_extractor::SmartIpKeyExtractor;
+use tower_http::services::ServeDir;
 use crate::api::sys_usage::exec_system_usage;
 use crate::repository::storage::get_geoip_path;
 use crate::utils::{exec_file_lock_prune, GeoIp};
@@ -315,6 +316,7 @@ pub async fn start_server(
     // Web Server
     let mut router = axum::Router::new()
         .route("/healthcheck", axum::routing::get(healthcheck))
+        .nest_service("/.well-known", ServeDir::new(web_dir_path.join("static/.well-known")))
         .merge(ws_api_register(
             web_auth_enabled,
             web_ui_path.as_str(),

backend/src/api/model/app_state.rs

@@ -2,10 +2,7 @@ use crate::api::config_watch::exec_config_watch;
 use crate::api::model::{ActiveProviderManager, ConnectionManager, EventManager, PlaylistStorage, PlaylistStorageState, SharedStreamManager};
 use crate::api::model::{ActiveUserManager, DownloadQueue};
 use crate::api::scheduler::exec_scheduler;
-use crate::model::{
-    AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets,
-    ScheduleConfig, SourcesConfig,
-};
+use crate::model::{AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets, ReverseProxyDisabledHeaderConfig, ScheduleConfig, SourcesConfig};
 use crate::repository::playlist_repository::load_target_into_memory_cache;
 use crate::tools::lru_cache::LRUResourceCache;
 use crate::utils::request::create_client;
@@ -444,6 +441,16 @@ impl AppState {
     pub async fn cache_playlist(&self, target_name: &str, playlist: PlaylistStorage) {
         self.playlists.cache_playlist(target_name, playlist).await;
     }
+
+    pub fn get_disabled_headers(&self) -> Option<ReverseProxyDisabledHeaderConfig> {
+        self
+            .app_config
+            .config
+            .load()
+            .reverse_proxy
+            .as_ref()
+            .and_then(|r| r.disabled_header.clone())
+    }
 }
 
 fn schedules_changed(a: &[ScheduleConfig], b: &[ScheduleConfig]) -> bool {

backend/src/api/model/streams/mod.rs

@@ -5,17 +5,18 @@ mod custom_video_stream;
 mod transport_stream_buffer;
 // mod chunked_buffer;
 mod provider_stream;
-mod persist_pipe_stream;
 mod provider_stream_factory;
 mod shared_stream_manager;
 mod active_client_stream;
 mod throttled_stream;
+pub mod persist_pipe_stream;
+
 pub(in crate) use self::transport_stream_buffer::*;
 pub(in crate::api) use self::provider_stream::*;
-pub(in crate::api) use self::persist_pipe_stream::*;
 pub(in crate::api) use self::provider_stream_factory::*;
 pub(in crate::api) use self::shared_stream_manager::*;
 pub(in crate::api) use self::active_client_stream::*;
 pub(in crate::api) use self::throttled_stream::*;
 pub(in crate::api) use self::timed_client_stream::*;
 pub(in crate::api) use self::custom_video_stream::*;
+pub use self::persist_pipe_stream::*;

backend/src/api/model/streams/persist_pipe_stream.rs

@@ -1,12 +1,13 @@
-use std::path::Path;
-use crate::api::model::StreamError;
+use crate::utils::request::DynReader;
+use crate::utils::{async_file_writer, IO_BUFFER_SIZE};
 use bytes::Bytes;
 use log::{debug, error};
+use std::path::{Path,};
 use std::sync::Arc;
-use tokio::io::AsyncWriteExt;
-use tokio_stream::{StreamExt};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio_stream::wrappers::ReceiverStream;
-use crate::utils::IO_BUFFER_SIZE;
+use tokio_stream::StreamExt;
+use crate::api::model::StreamError;
 
 pub fn tee_stream<S, W>(
     mut stream: S,
@@ -14,8 +15,9 @@ pub fn tee_stream<S, W>(
     file_path: &Path,
     callback: Arc<dyn Fn(usize) + Send + Sync>,
 ) -> ReceiverStream<Result<Bytes, StreamError>>
-where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin + 'static,
-      W: tokio::io::AsyncWrite + Send + Unpin + 'static,
+where
+    S: tokio_stream::Stream<Item=Result<Bytes, StreamError>> + Send + Unpin + 'static,
+    W: tokio::io::AsyncWrite + Send + Unpin + 'static,
 {
     let (tx, rx) = tokio::sync::mpsc::channel::<Result<Bytes, StreamError>>(32);
     let resource_path = file_path.to_owned();
@@ -77,3 +79,54 @@ where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin
 
     ReceiverStream::new(rx)
 }
+
+pub async fn tee_dyn_reader(
+    reader: DynReader,
+    persist_path: &Path,
+    callback: Option<Arc<dyn Fn(usize) + Send + Sync>>,
+) -> DynReader {
+    let file = match tokio::fs::File::create(persist_path).await {
+        Ok(f) => f,
+        Err(err) => {
+            error!("Cant open file to write: {}, {err}", persist_path.display());
+            return reader;
+        }
+    };
+
+    let (mut tx, rx) = tokio::io::duplex(IO_BUFFER_SIZE);
+    let mut writer = async_file_writer(file);
+    let reader_arc = reader;
+
+    tokio::spawn(async move {
+        let mut total_bytes = 0usize;
+        let mut buf = [0u8; 8192];
+
+        let mut reader = reader_arc;
+
+        loop {
+            let n = match reader.read(&mut buf).await {
+                Ok(0) | Err(_) => break,
+                Ok(n) => n,
+            };
+
+            total_bytes += n;
+
+            if tx.write_all(&buf[..n]).await.is_err() {
+                break;
+            }
+
+            if writer.write_all(&buf[..n]).await.is_err() {
+                break;
+            }
+        }
+
+        let _ = writer.flush().await;
+        let _ = tx.shutdown().await;
+
+        if let Some(cb) = callback {
+            cb(total_bytes);
+        }
+    });
+
+    Box::pin(rx) as DynReader
+}