xtream and m3u parsing with streaming

backend/src/api/model/app_state.rs

@@ -2,10 +2,7 @@ use crate::api::config_watch::exec_config_watch;
 use crate::api::model::{ActiveProviderManager, ConnectionManager, EventManager, PlaylistStorage, PlaylistStorageState, SharedStreamManager};
 use crate::api::model::{ActiveUserManager, DownloadQueue};
 use crate::api::scheduler::exec_scheduler;
-use crate::model::{
-    AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets,
-    ScheduleConfig, SourcesConfig,
-};
+use crate::model::{AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets, ReverseProxyDisabledHeaderConfig, ScheduleConfig, SourcesConfig};
 use crate::repository::playlist_repository::load_target_into_memory_cache;
 use crate::tools::lru_cache::LRUResourceCache;
 use crate::utils::request::create_client;
@@ -444,6 +441,16 @@ impl AppState {
     pub async fn cache_playlist(&self, target_name: &str, playlist: PlaylistStorage) {
         self.playlists.cache_playlist(target_name, playlist).await;
     }
+
+    pub fn get_disabled_headers(&self) -> Option<ReverseProxyDisabledHeaderConfig> {
+        self
+            .app_config
+            .config
+            .load()
+            .reverse_proxy
+            .as_ref()
+            .and_then(|r| r.disabled_header.clone())
+    }
 }
 
 fn schedules_changed(a: &[ScheduleConfig], b: &[ScheduleConfig]) -> bool {

backend/src/api/model/streams/mod.rs

@@ -5,17 +5,18 @@ mod custom_video_stream;
 mod transport_stream_buffer;
 // mod chunked_buffer;
 mod provider_stream;
-mod persist_pipe_stream;
 mod provider_stream_factory;
 mod shared_stream_manager;
 mod active_client_stream;
 mod throttled_stream;
+pub mod persist_pipe_stream;
+
 pub(in crate) use self::transport_stream_buffer::*;
 pub(in crate::api) use self::provider_stream::*;
-pub(in crate::api) use self::persist_pipe_stream::*;
 pub(in crate::api) use self::provider_stream_factory::*;
 pub(in crate::api) use self::shared_stream_manager::*;
 pub(in crate::api) use self::active_client_stream::*;
 pub(in crate::api) use self::throttled_stream::*;
 pub(in crate::api) use self::timed_client_stream::*;
 pub(in crate::api) use self::custom_video_stream::*;
+pub use self::persist_pipe_stream::*;

backend/src/api/model/streams/persist_pipe_stream.rs

@@ -1,12 +1,13 @@
-use std::path::Path;
-use crate::api::model::StreamError;
+use crate::utils::request::DynReader;
+use crate::utils::{async_file_writer, IO_BUFFER_SIZE};
 use bytes::Bytes;
 use log::{debug, error};
+use std::path::{Path,};
 use std::sync::Arc;
-use tokio::io::AsyncWriteExt;
-use tokio_stream::{StreamExt};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio_stream::wrappers::ReceiverStream;
-use crate::utils::IO_BUFFER_SIZE;
+use tokio_stream::StreamExt;
+use crate::api::model::StreamError;
 
 pub fn tee_stream<S, W>(
     mut stream: S,
@@ -14,8 +15,9 @@ pub fn tee_stream<S, W>(
     file_path: &Path,
     callback: Arc<dyn Fn(usize) + Send + Sync>,
 ) -> ReceiverStream<Result<Bytes, StreamError>>
-where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin + 'static,
-      W: tokio::io::AsyncWrite + Send + Unpin + 'static,
+where
+    S: tokio_stream::Stream<Item=Result<Bytes, StreamError>> + Send + Unpin + 'static,
+    W: tokio::io::AsyncWrite + Send + Unpin + 'static,
 {
     let (tx, rx) = tokio::sync::mpsc::channel::<Result<Bytes, StreamError>>(32);
     let resource_path = file_path.to_owned();
@@ -77,3 +79,54 @@ where S: tokio_stream::Stream<Item = Result<Bytes, StreamError>> + Send + Unpin
 
     ReceiverStream::new(rx)
 }
+
+pub async fn tee_dyn_reader(
+    reader: DynReader,
+    persist_path: &Path,
+    callback: Option<Arc<dyn Fn(usize) + Send + Sync>>,
+) -> DynReader {
+    let file = match tokio::fs::File::create(persist_path).await {
+        Ok(f) => f,
+        Err(err) => {
+            error!("Cant open file to write: {}, {err}", persist_path.display());
+            return reader;
+        }
+    };
+
+    let (mut tx, rx) = tokio::io::duplex(IO_BUFFER_SIZE);
+    let mut writer = async_file_writer(file);
+    let reader_arc = reader;
+
+    tokio::spawn(async move {
+        let mut total_bytes = 0usize;
+        let mut buf = [0u8; 8192];
+
+        let mut reader = reader_arc;
+
+        loop {
+            let n = match reader.read(&mut buf).await {
+                Ok(0) | Err(_) => break,
+                Ok(n) => n,
+            };
+
+            total_bytes += n;
+
+            if tx.write_all(&buf[..n]).await.is_err() {
+                break;
+            }
+
+            if writer.write_all(&buf[..n]).await.is_err() {
+                break;
+            }
+        }
+
+        let _ = writer.flush().await;
+        let _ = tx.shutdown().await;
+
+        if let Some(cb) = callback {
+            cb(total_bytes);
+        }
+    });
+
+    Box::pin(rx) as DynReader
+}