hls reverse proxy handling refactored

CHANGELOG.md

@@ -32,6 +32,7 @@ web_ui:
 You need to migrate the user db if you have used `use_user_db:true`. 
 Set it to `false` run old m3u-filter version, then update m3u-filter and set `use_user_db:true`and start.   
 - multi epg processing/optimization, auto guessing/assigning epg id's
+- fixed hls redirect url issue
 
 ```yaml
 epg:

Cargo.lock

@@ -1580,6 +1580,7 @@ name = "m3u-filter"
 version = "2.2.5"
 dependencies = [
  "axum",
+ "base64 0.22.1",
  "bincode",
  "blake3",
  "bytes",

Cargo.toml

@@ -50,7 +50,6 @@ bytes = "1.10"
 tokio-stream = { version = "0.1", features = ["sync"] }
 tokio = { version = "1.44", features = ["rt-multi-thread", "parking_lot", "fs"] }
 tokio-util = "0.7"
-paste = "1.0"
 tempfile = "3.19"
 ruzstd = "0"
 filetime = "0.2"
@@ -58,6 +57,8 @@ zeroize = "1.8"
 strsim = "0.11"
 rphonetic = "3"
 rayon = "1.10"
+base64 = "0"
+paste = "1"
 #[cfg(target_os = "macos")]
 libc = "0"
 #[cfg(target_os = "windows")]

src/api/endpoints/hls_api.rs

@@ -3,7 +3,7 @@ use crate::api::api_utils::try_option_bad_request;
 use crate::api::model::app_state::AppState;
 use crate::api::model::streams::provider_stream::{create_custom_video_stream_response, CustomVideoStreamType};
 use crate::model::api_proxy::ProxyUserCredentials;
-use crate::model::config::{ConfigInput, TargetType};
+use crate::model::config::{ConfigInput};
 use crate::model::playlist::{PlaylistItemType, XtreamCluster};
 use crate::processing::parser::hls::{rewrite_hls, RewriteHlsProps};
 use crate::utils::network::request;
@@ -12,44 +12,45 @@ use axum::response::IntoResponse;
 use log::{debug, error};
 use serde::Deserialize;
 use std::sync::Arc;
+use crate::utils::crypto_utils;
 
 #[derive(Debug, Deserialize)]
 struct HlsApiPathParams {
-    token: u32,
     username: String,
     password: String,
+    input_id: u16,
     stream_id: u32,
-    chunk: u32,
+    token: String,
+}
+
+fn hls_response(hls_content: String) -> impl IntoResponse + Send {
+        axum::response::Response::builder()
+            .status(axum::http::StatusCode::OK)
+            .header(axum::http::header::CONTENT_TYPE, "application/x-mpegURL")
+            .body(hls_content)
+            .unwrap()
+            .into_response()
 }
 
 pub(in crate::api) async fn handle_hls_stream_request(app_state: &Arc<AppState>,
                                                       user: &ProxyUserCredentials,
                                                       hls_url: &str,
                                                       virtual_id: u32,
-                                                      input: &ConfigInput,
-                                                      target_type: TargetType) -> impl axum::response::IntoResponse + Send {
+                                                      input: &ConfigInput) -> impl IntoResponse + Send {
     let url = replace_url_extension(hls_url, HLS_EXT);
     let server_info = app_state.config.get_user_server_info(user).await;
+
     match request::download_text_content(Arc::clone(&app_state.http_client), input, &url, None).await {
         Ok((content, response_url)) => {
-            let hls_token = app_state.hls_cache.new_token();
             let rewrite_hls_props = RewriteHlsProps {
                 base_url: &server_info.get_base_url(),
                 content: &content,
                 hls_url: response_url,
                 virtual_id,
-                token: hls_token,
-                target_type,
                 input_id: input.id,
             };
-            let (hls_entry, hls_content) = rewrite_hls(user, &rewrite_hls_props);
-            app_state.hls_cache.add_entry(hls_entry).await;
-            axum::response::Response::builder()
-                .status(axum::http::StatusCode::OK)
-                .header(axum::http::header::CONTENT_TYPE, "application/x-mpegurl")
-                .body(hls_content)
-                .unwrap()
-                .into_response()
+            let hls_content = rewrite_hls(user, &rewrite_hls_props);
+            hls_response(hls_content).into_response()
         }
         Err(err) => {
             error!("Failed to download m3u8 {}", sanitize_sensitive_info(err.to_string().as_str()));
@@ -73,29 +74,32 @@ async fn hls_api_stream(
         return create_custom_video_stream_response(&app_state.config, &CustomVideoStreamType::UserConnectionsExhausted).into_response();
     }
 
-    let Some(hls_entry) = app_state.hls_cache.get_entry(params.token).await else { return axum::http::StatusCode::BAD_REQUEST.into_response(); };
-    let Some(hls_url) = hls_entry.get_chunk_url(params.chunk) else { return axum::http::StatusCode::BAD_REQUEST.into_response(); };
+    let Ok(hls_url) = crypto_utils::decrypt_text(&params.token) else { return axum::http::StatusCode::BAD_REQUEST.into_response(); };
+
     let target_name = &target.name;
     let virtual_id = params.stream_id;
-    let input = try_option_bad_request!(app_state.config.get_input_by_id(hls_entry.input_id), true, format!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", XtreamCluster::Live));
+    let input = try_option_bad_request!(app_state.config.get_input_by_id(params.input_id), true, format!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", XtreamCluster::Live));
 
-    if is_hls_url(hls_url) {
-        return handle_hls_stream_request(&app_state, &user, hls_url, virtual_id, input, hls_entry.target_type).await.into_response();
+    if is_hls_url(&hls_url) {
+        return handle_hls_stream_request(&app_state, &user, &hls_url, virtual_id, input).await.into_response();
     }
 
-    // let (pli_url, input_name) = if hls_entry.target_type == TargetType::Xtream {
-    //     let (pli, _) = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None), true, format!("Failed to read xtream item for stream id {}", virtual_id));
-    //     (pli.url, pli.input_name)
-    // } else {
-    //     let pli = try_result_bad_request!(m3u_repository::m3u_get_item_for_stream_id(virtual_id, &app_state.config, target).await, true, format!("Failed to read xtream item for stream id {}", virtual_id));
-    //     (pli.url, pli.input_name)
-    // };
-    stream_response(&app_state, hls_url, &req_headers, Some(input), PlaylistItemType::LiveHls, target, &user).await.into_response()
+    stream_response(&app_state, &hls_url, &req_headers, Some(input), PlaylistItemType::LiveHls, target, &user).await.into_response()
 }
 
 pub fn hls_api_register() -> axum::Router<Arc<AppState>> {
     axum::Router::new()
-       .route("/hls/{token}/{username}/{password}/{stream_id}/{chunk}", axum::routing::get(hls_api_stream))
+        .route("/hls/{username}/{password}/{input_id}/{stream_id}/{token}", axum::routing::get(hls_api_stream))
     //cfg.service(web::resource("/hls/{token}/{stream}").route(web::get().to(xtream_player_api_hls_stream)));
     //cfg.service(web::resource("/play/{token}/{type}").route(web::get().to(xtream_player_api_play_stream)));
+}
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn test_hls_api_register() {
+
+    }
+
 }

src/api/endpoints/m3u_api.rs

@@ -105,7 +105,7 @@ async fn m3u_api_stream(
     if is_hls_request {
         let target_name = &target.name;
         let hls_input = try_option_bad_request!(input, true, format!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", XtreamCluster::Live));
-        return handle_hls_stream_request(&app_state, &user, &m3u_item.url, m3u_item.virtual_id, hls_input, TargetType::M3u).await.into_response();
+        return handle_hls_stream_request(&app_state, &user, &m3u_item.url, m3u_item.virtual_id, hls_input).await.into_response();
     }
 
     stream_response(&app_state, m3u_item.url.as_str(), &req_headers, input, m3u_item.item_type, target, &user).await.into_response()

src/api/main_api.rs

@@ -9,7 +9,6 @@ use crate::api::model::active_provider_manager::ActiveProviderManager;
 use crate::api::model::active_user_manager::ActiveUserManager;
 use crate::api::model::app_state::{AppState, HdHomerunAppState};
 use crate::api::model::download::DownloadQueue;
-use crate::api::model::hls_cache::HlsCache;
 use crate::api::model::streams::shared_stream_manager::SharedStreamManager;
 use crate::api::scheduler::start_scheduler;
 use crate::model::config::{validate_targets, Config, ProcessTargets, RateLimitConfig, ScheduleConfig};
@@ -138,7 +137,6 @@ fn create_shared_data(cfg: &Arc<Config>) -> AppState {
         http_client: Arc::new(client),
         downloads: Arc::new(DownloadQueue::new()),
         cache,
-        hls_cache: HlsCache::garbage_collected(),
         shared_stream_manager: Arc::new(SharedStreamManager::new()),
         active_users,
         active_provider,

src/api/model/app_state.rs

@@ -3,7 +3,6 @@ use std::sync::Arc;
 use crate::api::model::active_provider_manager::ActiveProviderManager;
 use crate::api::model::active_user_manager::ActiveUserManager;
 use crate::api::model::download::DownloadQueue;
-use crate::api::model::hls_cache::HlsCache;
 use crate::api::model::streams::shared_stream_manager::SharedStreamManager;
 use crate::model::api_proxy::UserConnectionPermission;
 use crate::model::config::{Config};
@@ -17,7 +16,6 @@ pub struct AppState {
     pub http_client: Arc<reqwest::Client>,
     pub downloads: Arc<DownloadQueue>,
     pub cache: Arc<Option<Mutex<LRUResourceCache>>>,
-    pub hls_cache: Arc<HlsCache>,
     pub shared_stream_manager: Arc<SharedStreamManager>,
     pub active_users: Arc<ActiveUserManager>,
     pub active_provider: Arc<ActiveProviderManager>,

src/api/model/hls_cache.rs

@@ -1,75 +0,0 @@
-use log::error;
-use std::collections::HashMap;
-use std::str::FromStr;
-use std::sync::Arc;
-use std::sync::atomic::AtomicU32;
-use std::time::Duration;
-use chrono::Local;
-use cron::Schedule;
-use tokio::sync::RwLock;
-use tokio::time::Instant;
-use crate::utils::sys_utils::exit;
-use crate::model::hls::HlsEntry;
-
-const EXPIRE_DURATION: u64 = 600; // 10 minutes
-const TOKEN_MAX: u32 = u32::MAX - u16::MAX  as u32;
-
-fn start_garbage_collector(cache: &Arc<HlsCache>) {
-    let cache_clone = cache.clone();
-    tokio::spawn(async move {
-        match Schedule::from_str("0 */15  *  *  *  *  *") {
-            Ok(schedule) => {
-                let offset = *Local::now().offset();
-                loop {
-                    let mut upcoming = schedule.upcoming(offset).take(1);
-                    if let Some(datetime) = upcoming.next() {
-                        tokio::time::sleep_until(tokio::time::Instant::from(crate::api::scheduler::datetime_to_instant(datetime))).await;
-                        cache_clone.gc().await;
-                    }
-                }
-            }
-            Err(err) => exit!("Failed to start scheduler: {}", err)
-        }
-    });
-}
-
-#[repr(align(64))]
-pub struct HlsCache {
-    pub entries: RwLock<HashMap<u32, HlsEntry>>,
-    counter:  AtomicU32,
-}
-
-impl HlsCache {
-    pub fn garbage_collected() -> Arc<Self> {
-        let cache = Arc::new(Self {
-            entries: RwLock::new(HashMap::new()),
-            counter: AtomicU32::new(1),
-        });
-
-        start_garbage_collector(&cache);
-        cache
-    }
-
-    pub async fn add_entry(&self, entry: HlsEntry) {
-        self.entries.write().await.insert(entry.token, entry);
-    }
-
-    pub async fn get_entry(&self, token: u32) -> Option<HlsEntry>{
-        self.entries.read().await.get(&token).cloned()
-    }
-
-    pub async fn gc(&self) {
-        let threshold = Instant::now() - Duration::from_secs(EXPIRE_DURATION);
-        // Remove all expired elements
-        self.entries.write().await.retain(|_, entry| entry.ts > threshold);
-    }
-
-    pub fn new_token(&self) -> u32 {
-        let token = self.counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-        if token > TOKEN_MAX {
-            self.counter.store(1, std::sync::atomic::Ordering::SeqCst);
-            return 1;
-        }
-        token
-    }
-}

src/api/model/mod.rs

@@ -8,5 +8,4 @@ pub(in crate::api) mod stream_error;
 pub(crate) mod streams;
 pub(in crate::api) mod active_user_manager;
 pub(in crate::api) mod active_provider_manager;
-pub(in crate::api) mod hls_cache;
 pub(in crate::api) mod stream;

src/model/hls.rs

@@ -1,20 +0,0 @@
-use std::collections::HashMap;
-use tokio::time::Instant;
-use crate::model::config::TargetType;
-
-#[derive(Clone, Debug)]
-pub struct HlsEntry {
-    pub ts: Instant,
-    pub token: u32,
-    pub target_type: TargetType,
-    pub input_id: u16,
-    pub virtual_id: u32,
-    pub chunk: u32,
-    pub chunks: HashMap<u32, String>,
-}
-
-impl HlsEntry {
-    pub fn get_chunk_url(&self, chunk: u32) -> Option<&String> {
-       self.chunks.get(&chunk)
-    }
-}

src/model/mod.rs

@@ -7,5 +7,4 @@ pub mod xmltv;
 pub mod xtream;
 pub mod healthcheck;
 pub mod playlist_categories;
-pub mod hdhomerun_config;
-pub mod hls;
+pub mod hdhomerun_config;

src/processing/parser/hls.rs

@@ -1,9 +1,6 @@
 use crate::model::api_proxy::ProxyUserCredentials;
-use crate::model::config::TargetType;
-use crate::model::hls::HlsEntry;
-use std::collections::HashMap;
+use crate::utils::crypto_utils::encrypt_text;
 use std::str;
-use tokio::time::Instant;
 
 pub const HLS_PREFIX: &str = "hls";
 
@@ -13,9 +10,7 @@ pub struct RewriteHlsProps<'a> {
     pub content: &'a str,
     pub hls_url: String,
     pub virtual_id: u32,
-    pub token: u32,
-    pub target_type: TargetType,
-    pub input_id: u16
+    pub input_id: u16,
 }
 
 
@@ -23,52 +18,32 @@ fn rewrite_hls_url(input: &str, replacement: &str) -> String {
     if replacement.starts_with('/') {
         let parts = input.splitn(4, '/').collect::<Vec<&str>>();
         if parts.len() < 4 {
-            return replacement.to_string(); // wenn weniger als 3 Slashes → alles ersetzen
+            return replacement.to_string(); // less than 3 Slashes → replace all
         }
         format!("{}/{}/{}{}", parts[0], parts[1], parts[2], replacement)
     } else {
         match input.rsplitn(2, '/').collect::<Vec<&str>>().as_slice() {
-            [_after, before] => format!("{}/{}", before, replacement),
+            [_after, before] => format!("{before}/{replacement}"),
             [_only] => replacement.to_string(),
             _ => input.to_string(),
         }
     }
-    // match input.rsplitn(2, '/').collect::<Vec<&str>>().as_slice() {
-    //     [_after, before] => format!("{before}/{replacement}"),
-    //     [_only] => replacement.to_string(), // if there is no slash, replace complete
-    //     _ => input.to_string(), // fallback, should never happen
-    // }
 }
 
-pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps ) -> (HlsEntry, String) {
+pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps) -> String {
     let username = &user.username;
     let password = &user.password;
-    let mut chunk: u32 = 1;
-    let mut chunks = HashMap::new();
     let mut result = Vec::new();
     for line in props.content.lines() {
         if line.starts_with('#') {
             result.push(line.to_string());
+        } else if let Ok(token) = if line.starts_with("http") {
+            encrypt_text(line)
         } else {
-            let url = if line.starts_with("http") {
-                line.to_string()
-            } else {
-               rewrite_hls_url(&props.hls_url, line)
-            };
-            chunks.insert(chunk, url);
-            result.push(format!("{}/{HLS_PREFIX}/{}/{username}/{password}/{}/{chunk}", props.base_url, props.token, props.virtual_id));
-            chunk += 1;
+            encrypt_text(&rewrite_hls_url(&props.hls_url, line))
+        } {
+            result.push(format!("{}/{HLS_PREFIX}/{username}/{password}/{}/{}/{token}", props.base_url, props.input_id, props.virtual_id));
         }
     }
-
-    let hls = HlsEntry {
-        ts: Instant::now(),
-        token: props.token,
-        target_type: props.target_type,
-        input_id: props.input_id,
-        virtual_id: props.virtual_id,
-        chunk,
-        chunks,
-    };
-    (hls, result.join("\r\n"))
+    result.join("\r\n")
 }

src/repository/bplustree.rs

@@ -780,10 +780,10 @@ mod tests {
         data: String,
     }
 
-    use std::time::{SystemTime, UNIX_EPOCH};
+    use crate::utils::time_utils::current_time_secs;
 
     fn generate_random_string(length: usize) -> String {
-        let mut rng = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+        let mut rng = current_time_secs();
         let charset = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
 
         let mut random_string = String::with_capacity(length);

src/utils/crypto_utils.rs

@@ -0,0 +1,50 @@
+use openssl::symm::{Cipher, Crypter, Mode};
+use base64::{engine::general_purpose, Engine as _};
+use rand::Rng;
+use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
+
+const SECRET_KEY: &[u8; 16] = b"my-secret-32-byt"; // 32 bytes = AES-256
+
+pub fn encrypt_text(text: &str) -> Result<String, M3uFilterError> {
+    let iv: [u8; 16] = rand::rng().random(); // Random IV (AES-CBC 16 Bytes)
+    let cipher = Cipher::aes_128_cbc();
+
+    let mut crypter = Crypter::new(cipher, Mode::Encrypt, SECRET_KEY, Some(&iv)).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))?;
+    let mut ciphertext = vec![0; text.len() + cipher.block_size()];
+    let mut count = crypter.update(text.as_bytes(), &mut ciphertext).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))?;
+    count += crypter.finalize(&mut ciphertext[count..]).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))?;
+    ciphertext.truncate(count);
+
+    // IV + Ciphertext
+    let mut out = iv.to_vec();
+    out.extend(ciphertext);
+    Ok(general_purpose::URL_SAFE_NO_PAD.encode(out))
+}
+
+pub fn decrypt_text(encrypted_text: &str) -> Result<String, M3uFilterError> {
+    let data = general_purpose::URL_SAFE_NO_PAD.decode(encrypted_text).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))?;
+    let (iv, ciphertext) = data.split_at(16); // first 16 bytes IV
+    let cipher = Cipher::aes_128_cbc();
+
+    let mut crypter = Crypter::new(cipher, Mode::Decrypt, SECRET_KEY, Some(iv)).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))?;
+    let mut decrypted = vec![0; ciphertext.len() + cipher.block_size()];
+    let mut count = crypter.update(ciphertext, &mut decrypted).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))?;
+    count += crypter.finalize(&mut decrypted[count..]).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))?;
+    decrypted.truncate(count);
+
+    String::from_utf8(decrypted).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::utils::crypto_utils::{decrypt_text, encrypt_text};
+
+    #[test]
+    fn test_encrypt() {
+        let plain = "hello world";
+        let encrypted = encrypt_text(&plain);
+        let decrypted = decrypt_text(&encrypted.unwrap()).unwrap();
+
+        assert_eq!(decrypted, plain);
+    }
+}

src/utils/mod.rs

@@ -8,6 +8,8 @@ pub mod compression;
 pub(crate) mod file;
 pub(crate) mod network;
 pub mod bincode_utils;
+pub mod time_utils;
+pub mod crypto_utils;
 
 #[macro_export]
 macro_rules! debug_if_enabled {

src/utils/network/request.rs

@@ -238,7 +238,9 @@ async fn get_remote_content(client: Arc<reqwest::Client>, input: &ConfigInput, u
             let is_success = response.status().is_success();
             if is_success {
                 let response_url = response.url().to_string();
-                let header_value = response.headers().get(CONTENT_ENCODING);
+                let headers = response.headers();
+                debug!("{headers:?}");
+                let header_value = headers.get(CONTENT_ENCODING);
                 let mut encoding = header_value.and_then(|encoding_header| encoding_header.to_str().map_or(None, |value| Some(value.to_string())));
                 match response.bytes().await {
                     Ok(bytes) => {
@@ -439,7 +441,7 @@ pub fn classify_content_type(headers: &[(String, String)]) -> MimeCategory {
         })
         .map_or(MimeCategory::Unknown, |v| match v.to_lowercase().as_str() {
             v if v.starts_with("video/") || v == "application/octet-stream" => MimeCategory::Video,
-            "application/vnd.apple.mpegurl" | "application/x-mpegurl" => MimeCategory::M3U8,
+            v if v.contains("mpegurl") => MimeCategory::M3U8,
             v if v.starts_with("image/") => MimeCategory::Image,
             v if v.starts_with("application/json") || v.ends_with("+json") => MimeCategory::Json,
             v if v.starts_with("application/xml") || v.ends_with("+xml") || v == "text/xml" => MimeCategory::Xml,

src/utils/time_utils.rs

@@ -0,0 +1,8 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+pub fn current_time_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}

test/rest-api.http

@@ -36,7 +36,7 @@ GET {{local}}/player_api.php?username={{username}}&password={{password}}
 GET {{local}}/player_api.php?username={{username}}&password={{password}}&action=get_live_categories
 
 ### xtream live_streams
-GET {{local}}/player_api.php?username={{username}}&password={{password}}&action=get_live_streams&category_id=7
+GET {{local}}/player_api.php?username={{username}}&password={{password}}&action=get_live_streams
 
 ### xtream vod_categories
 GET {{local}}/player_api.php?username={{username}}&password={{password}}&action=get_vod_categories